lib/readability/Readability.php

   1 <?php
   2 /**
   3 * Arc90's Readability ported to PHP for FiveFilters.org
   4 * Based on readability.js version 1.7.1 (without multi-page support)
   5 * Updated to allow HTML5 parsing with html5lib
   6 * Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
   7 * ------------------------------------------------------
   8 * Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
   9 * Arc90's project URL: http://lab.arc90.com/experiments/readability/
  10 * JS Source: http://code.google.com/p/arc90labs-readability
  11 * Ported by: Keyvan Minoukadeh, http://www.keyvan.net
  12 * More information: http://fivefilters.org/content-only/
  13 * License: Apache License, Version 2.0
  14 * Requires: PHP5
  15 * Date: 2012-09-19
  16 *
  17 * Differences between the PHP port and the original
  18 * ------------------------------------------------------
  19 * Arc90's Readability is designed to run in the browser. It works on the DOM
  20 * tree (the parsed HTML) after the page's CSS styles have been applied and
  21 * Javascript code executed. This PHP port does not run inside a browser.
  22 * We use PHP's ability to parse HTML to build our DOM tree, but we cannot
  23 * rely on CSS or Javascript support. As such, the results will not always
  24 * match Arc90's Readability. (For example, if a web page contains CSS style
  25 * rules or Javascript code which hide certain HTML elements from display,
  26 * Arc90's Readability will dismiss those from consideration but our PHP port,
  27 * unable to understand CSS or Javascript, will not know any better.)
  28 *
  29 * Another significant difference is that the aim of Arc90's Readability is
  30 * to re-present the main content block of a given web page so users can
  31 * read it more easily in their browsers. Correct identification, clean up,
  32 * and separation of the content block is only a part of this process.
  33 * This PHP port is only concerned with this part, it does not include code
  34 * that relates to presentation in the browser - Arc90 already do
  35 * that extremely well, and for PDF output there's FiveFilters.org's
  36 * PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
  37 *
  38 * Finally, this class contains methods that might be useful for developers
  39 * working on HTML document fragments. So without deviating too much from
  40 * the original code (which I don't want to do because it makes debugging
  41 * and updating more difficult), I've tried to make it a little more
  42 * developer friendly. You should be able to use the methods here on
  43 * existing DOMElement objects without passing an entire HTML document to
  44 * be parsed.
  45 */
  46
  47 // This class allows us to do JavaScript like assignements to innerHTML
  48 require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');
  49 libxml_use_internal_errors(true);
  50
  51 // Alternative usage (for testing only!)
  52 // uncomment the lines below and call Readability.php in your browser
  53 // passing it the URL of the page you'd like content from, e.g.:
  54 // Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php
  55
  56 /*
  57 if (!isset($_GET['url']) || $_GET['url'] == '') {
  58         die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
  59 }
  60 $url = $_GET['url'];
  61 if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
  62 $html = file_get_contents($url);
  63 $r = new Readability($html, $url);
  64 $r->init();
  65 echo $r->articleContent->innerHTML;
  66 */
  67
  68 class Readability
  69 {
  70         public $version = '1.7.1-without-multi-page';
  71         public $convertLinksToFootnotes = false;
  72         public $revertForcedParagraphElements = true;
  73         public $articleTitle;
  74         public $articleContent;
  75         public $dom;
  76         public $url = null; // optional - URL where HTML was retrieved
  77         public $debug = false;
  78         public $lightClean = true; // preserves more content (experimental) added 2012-09-19
  79         protected $body = null; //
  80         protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
  81         protected $flags = 7; // 1 | 2 | 4;   // Start with all flags set.
  82         protected $success = false; // indicates whether we were able to extract or not
  83
  84         /**
  85         * All of the regular expressions in use within readability.
  86         * Defined up here so we don't instantiate them repeatedly in loops.
  87         **/
  88         public $regexps = array(
  89                 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
  90                 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
  91                 'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
  92                 'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
  93                 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
  94                 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
  95                 'replaceFonts' => '/<(\/?)font[^>]*>/i',
  96                 // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
  97                 'normalize' => '/\s{2,}/',
  98                 'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
  99                 'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
 100                 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
 101         );
 102
 103         /* constants */
 104         const FLAG_STRIP_UNLIKELYS = 1;
 105         const FLAG_WEIGHT_CLASSES = 2;
 106         const FLAG_CLEAN_CONDITIONALLY = 4;
 107
 108         /**
 109         * Create instance of Readability
 110         * @param string UTF-8 encoded string
 111         * @param string (optional) URL associated with HTML (used for footnotes)
 112         * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
 113         */
 114         function __construct($html, $url=null, $parser='libxml')
 115         {
 116                 $this->url = $url;
 117                 /* Turn all double br's into p's */
 118                 $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
 119                 $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
 120                 $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
 121                 if (trim($html) == '') $html = '<html></html>';
 122                 if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
 123                         // all good
 124                 } else {
 125                         $this->dom = new DOMDocument();
 126                         $this->dom->preserveWhiteSpace = false;
 127                         @$this->dom->loadHTML($html);
 128                 }
 129                 $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
 130         }
 131
 132         /**
 133         * Get article title element
 134         * @return DOMElement
 135         */
 136         public function getTitle() {
 137                 return $this->articleTitle;
 138         }
 139
 140         /**
 141         * Get article content element
 142         * @return DOMElement
 143         */
 144         public function getContent() {
 145                 return $this->articleContent;
 146         }
 147
 148         /**
 149         * Runs readability.
 150         *
 151         * Workflow:
 152         *  1. Prep the document by removing script tags, css, etc.
 153         *  2. Build readability's DOM tree.
 154         *  3. Grab the article content from the current dom tree.
 155         *  4. Replace the current DOM tree with the new one.
 156         *  5. Read peacefully.
 157         *
 158         * @return boolean true if we found content, false otherwise
 159         **/
 160         public function init()
 161         {
 162                 if (!isset($this->dom->documentElement)) return false;
 163                 $this->removeScripts($this->dom);
 164                 //die($this->getInnerHTML($this->dom->documentElement));
 165
 166                 // Assume successful outcome
 167                 $this->success = true;
 168
 169                 $bodyElems = $this->dom->getElementsByTagName('body');
 170                 if ($bodyElems->length > 0) {
 171                         if ($this->bodyCache == null) {
 172                                 $this->bodyCache = $bodyElems->item(0)->innerHTML;
 173                         }
 174                         if ($this->body == null) {
 175                                 $this->body = $bodyElems->item(0);
 176                         }
 177                 }
 178
 179                 $this->prepDocument();
 180
 181                 //die($this->dom->documentElement->parentNode->nodeType);
 182                 //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));
 183                 //die($this->getInnerHTML($this->dom->documentElement));
 184
 185                 /* Build readability's DOM tree */
 186                 $overlay        = $this->dom->createElement('div');
 187                 $innerDiv       = $this->dom->createElement('div');
 188                 $articleTitle   = $this->getArticleTitle();
 189                 $articleContent = $this->grabArticle();
 190
 191                 if (!$articleContent) {
 192                         $this->success = false;
 193                         $articleContent = $this->dom->createElement('div');
 194                         $articleContent->setAttribute('id', 'readability-content');
 195                         $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
 196                 }
 197
 198                 $overlay->setAttribute('id', 'readOverlay');
 199                 $innerDiv->setAttribute('id', 'readInner');
 200
 201                 /* Glue the structure of our document together. */
 202                 $innerDiv->appendChild($articleTitle);
 203                 $innerDiv->appendChild($articleContent);
 204                 $overlay->appendChild($innerDiv);
 205
 206                 /* Clear the old HTML, insert the new content. */
 207                 $this->body->innerHTML = '';
 208                 $this->body->appendChild($overlay);
 209                 //document.body.insertBefore(overlay, document.body.firstChild);
 210                 $this->body->removeAttribute('style');
 211
 212                 $this->postProcessContent($articleContent);
 213
 214                 // Set title and content instance variables
 215                 $this->articleTitle = $articleTitle;
 216                 $this->articleContent = $articleContent;
 217
 218                 return $this->success;
 219         }
 220
 221         /**
 222         * Debug
 223         */
 224         protected function dbg($msg) {
 225                 if ($this->debug) echo '* ',$msg, "\n";
 226         }
 227
 228         /**
 229         * Run any post-process modifications to article content as necessary.
 230         *
 231         * @param DOMElement
 232         * @return void
 233         */
 234         public function postProcessContent($articleContent) {
 235                 if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) {
 236                         $this->addFootnotes($articleContent);
 237                 }
 238         }
 239
 240         /**
 241         * Get the article title as an H1.
 242         *
 243         * @return DOMElement
 244         */
 245         protected function getArticleTitle() {
 246                 $curTitle = '';
 247                 $origTitle = '';
 248
 249                 try {
 250                         $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
 251                 } catch(Exception $e) {}
 252
 253                 if (preg_match('/ [\|\-] /', $curTitle))
 254                 {
 255                         $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
 256
 257                         if (count(explode(' ', $curTitle)) < 3) {
 258                                 $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);
 259                         }
 260                 }
 261                 else if (strpos($curTitle, ': ') !== false)
 262                 {
 263                         $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);
 264
 265                         if (count(explode(' ', $curTitle)) < 3) {
 266                                 $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);
 267                         }
 268                 }
 269                 else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)
 270                 {
 271                         $hOnes = $this->dom->getElementsByTagName('h1');
 272                         if($hOnes->length == 1)
 273                         {
 274                                 $curTitle = $this->getInnerText($hOnes->item(0));
 275                         }
 276                 }
 277
 278                 $curTitle = trim($curTitle);
 279
 280                 if (count(explode(' ', $curTitle)) <= 4) {
 281                         $curTitle = $origTitle;
 282                 }
 283
 284                 $articleTitle = $this->dom->createElement('h1');
 285                 $articleTitle->innerHTML = $curTitle;
 286
 287                 return $articleTitle;
 288         }
 289
 290         /**
 291         * Prepare the HTML document for readability to scrape it.
 292         * This includes things like stripping javascript, CSS, and handling terrible markup.
 293         *
 294         * @return void
 295         **/
 296         protected function prepDocument() {
 297                 /**
 298                 * In some cases a body element can't be found (if the HTML is totally hosed for example)
 299                 * so we create a new body node and append it to the document.
 300                 */
 301                 if ($this->body == null)
 302                 {
 303                         $this->body = $this->dom->createElement('body');
 304                         $this->dom->documentElement->appendChild($this->body);
 305                 }
 306                 $this->body->setAttribute('id', 'readabilityBody');
 307
 308                 /* Remove all style tags in head */
 309                 $styleTags = $this->dom->getElementsByTagName('style');
 310                 for ($i = $styleTags->length-1; $i >= 0; $i--)
 311                 {
 312                         $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
 313                 }
 314
 315                 /* Turn all double br's into p's */
 316                 /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
 317                 //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');
 318                 // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.
 319                 // Manipulating innerHTML as it's done in JS is not possible in PHP.
 320         }
 321
 322         /**
 323         * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
 324         * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
 325         *
 326         * @return void
 327         **/
 328         public function addFootnotes($articleContent) {
 329                 $footnotesWrapper = $this->dom->createElement('div');
 330                 $footnotesWrapper->setAttribute('id', 'readability-footnotes');
 331                 $footnotesWrapper->innerHTML = '<h3>References</h3>';
 332
 333                 $articleFootnotes = $this->dom->createElement('ol');
 334                 $articleFootnotes->setAttribute('id', 'readability-footnotes-list');
 335                 $footnotesWrapper->appendChild($articleFootnotes);
 336
 337                 $articleLinks = $articleContent->getElementsByTagName('a');
 338
 339                 $linkCount = 0;
 340                 for ($i = 0; $i < $articleLinks->length; $i++)
 341                 {
 342                         $articleLink  = $articleLinks->item($i);
 343                         $footnoteLink = $articleLink->cloneNode(true);
 344                         $refLink      = $this->dom->createElement('a');
 345                         $footnote     = $this->dom->createElement('li');
 346                         $linkDomain   = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
 347                         if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);
 348                         //linkDomain   = footnoteLink.host ? footnoteLink.host : document.location.host,
 349                         $linkText     = $this->getInnerText($articleLink);
 350
 351                         if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
 352                                 continue;
 353                         }
 354
 355                         $linkCount++;
 356
 357                         /** Add a superscript reference after the article link */
 358                         $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);
 359                         $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';
 360                         $refLink->setAttribute('class', 'readability-DoNotFootnote');
 361                         $refLink->setAttribute('style', 'color: inherit;');
 362
 363                         //TODO: does this work or should we use DOMNode.isSameNode()?
 364                         if ($articleLink->parentNode->lastChild == $articleLink) {
 365                                 $articleLink->parentNode->appendChild($refLink);
 366                         } else {
 367                                 $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);
 368                         }
 369
 370                         $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
 371                         $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
 372
 373                         $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';
 374
 375                         $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);
 376                         $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
 377
 378                         $footnote->appendChild($footnoteLink);
 379                         if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';
 380
 381                         $articleFootnotes->appendChild($footnote);
 382                 }
 383
 384                 if ($linkCount > 0) {
 385                         $articleContent->appendChild($footnotesWrapper);
 386                 }
 387         }
 388
 389         /**
 390         * Reverts P elements with class 'readability-styled'
 391         * to text nodes - which is what they were before.
 392         *
 393         * @param DOMElement
 394         * @return void
 395         */
 396         function revertReadabilityStyledElements($articleContent) {
 397                 $xpath = new DOMXPath($articleContent->ownerDocument);
 398                 $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);
 399                 //$elems = $articleContent->getElementsByTagName('p');
 400                 for ($i = $elems->length-1; $i >= 0; $i--) {
 401                         $e = $elems->item($i);
 402                         $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
 403                         //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {
 404                         //      $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);
 405                         //}
 406                 }
 407         }
 408
 409         /**
 410         * Prepare the article node for display. Clean out any inline styles,
 411         * iframes, forms, strip extraneous <p> tags, etc.
 412         *
 413         * @param DOMElement
 414         * @return void
 415         */
 416         function prepArticle($articleContent) {
 417                 $this->cleanStyles($articleContent);
 418                 $this->killBreaks($articleContent);
 419                 if ($this->revertForcedParagraphElements) {
 420                         $this->revertReadabilityStyledElements($articleContent);
 421                 }
 422
 423                 /* Clean out junk from the article content */
 424                 $this->cleanConditionally($articleContent, 'form');
 425                 $this->clean($articleContent, 'object');
 426                 $this->clean($articleContent, 'h1');
 427
 428                 /**
 429                 * If there is only one h2, they are probably using it
 430                 * as a header and not a subheader, so remove it since we already have a header.
 431                 ***/
 432                 if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
 433                         $this->clean($articleContent, 'h2');
 434                 }
 435                 $this->clean($articleContent, 'iframe');
 436
 437                 $this->cleanHeaders($articleContent);
 438
 439                 /* Do these last as the previous stuff may have removed junk that will affect these */
 440                 $this->cleanConditionally($articleContent, 'table');
 441                 $this->cleanConditionally($articleContent, 'ul');
 442                 $this->cleanConditionally($articleContent, 'div');
 443
 444                 /* Remove extra paragraphs */
 445                 $articleParagraphs = $articleContent->getElementsByTagName('p');
 446                 for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
 447                 {
 448                         $imgCount    = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
 449                         $embedCount  = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
 450                         $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
 451                         $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
 452
 453                         if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
 454                         {
 455                                 $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
 456                         }
 457                 }
 458
 459                 try {
 460                         $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);
 461                         //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');
 462                 }
 463                 catch (Exception $e) {
 464                         $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
 465                 }
 466         }
 467
 468         /**
 469         * Initialize a node with the readability object. Also checks the
 470         * className/id for special names to add to its score.
 471         *
 472         * @param Element
 473         * @return void
 474         **/
 475         protected function initializeNode($node) {
 476                 $readability = $this->dom->createAttribute('readability');
 477                 $readability->value = 0; // this is our contentScore
 478                 $node->setAttributeNode($readability);
 479
 480                 switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
 481                         case 'DIV':
 482                                 $readability->value += 5;
 483                                 break;
 484
 485                         case 'PRE':
 486                         case 'TD':
 487                         case 'BLOCKQUOTE':
 488                                 $readability->value += 3;
 489                                 break;
 490
 491                         case 'ADDRESS':
 492                         case 'OL':
 493                         case 'UL':
 494                         case 'DL':
 495                         case 'DD':
 496                         case 'DT':
 497                         case 'LI':
 498                         case 'FORM':
 499                                 $readability->value -= 3;
 500                                 break;
 501
 502                         case 'H1':
 503                         case 'H2':
 504                         case 'H3':
 505                         case 'H4':
 506                         case 'H5':
 507                         case 'H6':
 508                         case 'TH':
 509                                 $readability->value -= 5;
 510                                 break;
 511                 }
 512                 $readability->value += $this->getClassWeight($node);
 513         }
 514
 515         /***
 516         * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
 517         *               most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
 518         *
 519         * @return DOMElement
 520         **/
 521         protected function grabArticle($page=null) {
 522                 $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
 523                 if (!$page) $page = $this->dom;
 524                 $allElements = $page->getElementsByTagName('*');
 525                 /**
 526                 * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
 527                 * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
 528                 *
 529                 * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
 530                 * TODO: Shouldn't this be a reverse traversal?
 531                 **/
 532                 $node = null;
 533                 $nodesToScore = array();
 534                 for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
 535                 //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
 536                         //$node = $targetList->item($nodeIndex);
 537                         $tagName = strtoupper($node->tagName);
 538                         /* Remove unlikely candidates */
 539                         if ($stripUnlikelyCandidates) {
 540                                 $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
 541                                 if (
 542                                         preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
 543                                         !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
 544                                         $tagName != 'BODY'
 545                                 )
 546                                 {
 547                                         $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
 548                                         //$nodesToRemove[] = $node;
 549                                         $node->parentNode->removeChild($node);
 550                                         $nodeIndex--;
 551                                         continue;
 552                                 }
 553                         }
 554
 555                         if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
 556                                 $nodesToScore[] = $node;
 557                         }
 558
 559                         /* Turn all divs that don't have children block level elements into p's */
 560                         if ($tagName == 'DIV') {
 561                                 if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
 562                                         //$this->dbg('Altering div to p');
 563                                         $newNode = $this->dom->createElement('p');
 564                                         try {
 565                                                 $newNode->innerHTML = $node->innerHTML;
 566                                                 //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
 567                                                 $node->parentNode->replaceChild($newNode, $node);
 568                                                 $nodeIndex--;
 569                                                 $nodesToScore[] = $node; // or $newNode?
 570                                         }
 571                                         catch(Exception $e) {
 572                                                 $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
 573                                         }
 574                                 }
 575                                 else
 576                                 {
 577                                         /* EXPERIMENTAL */
 578                                         // TODO: change these p elements back to text nodes after processing
 579                                         for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
 580                                                 $childNode = $node->childNodes->item($i);
 581                                                 if ($childNode->nodeType == 3) { // XML_TEXT_NODE
 582                                                         //$this->dbg('replacing text node with a p tag with the same content.');
 583                                                         $p = $this->dom->createElement('p');
 584                                                         $p->innerHTML = $childNode->nodeValue;
 585                                                         $p->setAttribute('style', 'display: inline;');
 586                                                         $p->setAttribute('class', 'readability-styled');
 587                                                         $childNode->parentNode->replaceChild($p, $childNode);
 588                                                 }
 589                                         }
 590                                 }
 591                         }
 592                 }
 593
 594                 /**
 595                 * Loop through all paragraphs, and assign a score to them based on how content-y they look.
 596                 * Then add their score to their parent node.
 597                 *
 598                 * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
 599                 **/
 600                 $candidates = array();
 601                 for ($pt=0; $pt < count($nodesToScore); $pt++) {
 602                         $parentNode      = $nodesToScore[$pt]->parentNode;
 603                         // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
 604                         $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
 605                         $innerText       = $this->getInnerText($nodesToScore[$pt]);
 606
 607                         if (!$parentNode || !isset($parentNode->tagName)) {
 608                                 continue;
 609                         }
 610
 611                         /* If this paragraph is less than 25 characters, don't even count it. */
 612                         if(strlen($innerText) < 25) {
 613                                 continue;
 614                         }
 615
 616                         /* Initialize readability data for the parent. */
 617                         if (!$parentNode->hasAttribute('readability'))
 618                         {
 619                                 $this->initializeNode($parentNode);
 620                                 $candidates[] = $parentNode;
 621                         }
 622
 623                         /* Initialize readability data for the grandparent. */
 624                         if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
 625                         {
 626                                 $this->initializeNode($grandParentNode);
 627                                 $candidates[] = $grandParentNode;
 628                         }
 629
 630                         $contentScore = 0;
 631
 632                         /* Add a point for the paragraph itself as a base. */
 633                         $contentScore++;
 634
 635                         /* Add points for any commas within this paragraph */
 636                         $contentScore += count(explode(',', $innerText));
 637
 638                         /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
 639                         $contentScore += min(floor(strlen($innerText) / 100), 3);
 640
 641                         /* Add the score to the parent. The grandparent gets half. */
 642                         $parentNode->getAttributeNode('readability')->value += $contentScore;
 643
 644                         if ($grandParentNode) {
 645                                 $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;
 646                         }
 647                 }
 648
 649                 /**
 650                 * After we've calculated scores, loop through all of the possible candidate nodes we found
 651                 * and find the one with the highest score.
 652                 **/
 653                 $topCandidate = null;
 654                 for ($c=0, $cl=count($candidates); $c < $cl; $c++)
 655                 {
 656                         /**
 657                         * Scale the final candidates score based on link density. Good content should have a
 658                         * relatively small link density (5% or less) and be mostly unaffected by this operation.
 659                         **/
 660                         $readability = $candidates[$c]->getAttributeNode('readability');
 661                         $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
 662
 663                         $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
 664
 665                         if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
 666                                 $topCandidate = $candidates[$c];
 667                         }
 668                 }
 669
 670                 /**
 671                 * If we still have no top candidate, just use the body as a last resort.
 672                 * We also have to copy the body node so it is something we can modify.
 673                 **/
 674                 if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
 675                 {
 676                         $topCandidate = $this->dom->createElement('div');
 677                         if ($page instanceof DOMDocument) {
 678                                 if (!isset($page->documentElement)) {
 679                                         // we don't have a body either? what a mess! :)
 680                                 } else {
 681                                         $topCandidate->innerHTML = $page->documentElement->innerHTML;
 682                                         $page->documentElement->innerHTML = '';
 683                                         $this->reinitBody();
 684                                         $page->documentElement->appendChild($topCandidate);
 685                                 }
 686                         } else {
 687                                 $topCandidate->innerHTML = $page->innerHTML;
 688                                 $page->innerHTML = '';
 689                                 $page->appendChild($topCandidate);
 690                         }
 691                         $this->initializeNode($topCandidate);
 692                 }
 693
 694                 /**
 695                 * Now that we have the top candidate, look through its siblings for content that might also be related.
 696                 * Things like preambles, content split by ads that we removed, etc.
 697                 **/
 698                 $articleContent        = $this->dom->createElement('div');
 699                 $articleContent->setAttribute('id', 'readability-content');
 700                 $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
 701                 $siblingNodes          = @$topCandidate->parentNode->childNodes;
 702                 if (!isset($siblingNodes)) {
 703                         $siblingNodes = new stdClass;
 704                         $siblingNodes->length = 0;
 705                 }
 706
 707                 for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
 708                 {
 709                         $siblingNode = $siblingNodes->item($s);
 710                         $append      = false;
 711
 712                         $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
 713
 714                         //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
 715
 716                         if ($siblingNode === $topCandidate)
 717                         // or if ($siblingNode->isSameNode($topCandidate))
 718                         {
 719                                 $append = true;
 720                         }
 721
 722                         $contentBonus = 0;
 723                         /* Give a bonus if sibling nodes and top candidates have the example same classname */
 724                         if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
 725                                 $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
 726                         }
 727
 728                         if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
 729                         {
 730                                 $append = true;
 731                         }
 732
 733                         if (strtoupper($siblingNode->nodeName) == 'P') {
 734                                 $linkDensity = $this->getLinkDensity($siblingNode);
 735                                 $nodeContent = $this->getInnerText($siblingNode);
 736                                 $nodeLength  = strlen($nodeContent);
 737
 738                                 if ($nodeLength > 80 && $linkDensity < 0.25)
 739                                 {
 740                                         $append = true;
 741                                 }
 742                                 else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
 743                                 {
 744                                         $append = true;
 745                                 }
 746                         }
 747
 748                         if ($append)
 749                         {
 750                                 $this->dbg('Appending node: ' . $siblingNode->nodeName);
 751
 752                                 $nodeToAppend = null;
 753                                 $sibNodeName = strtoupper($siblingNode->nodeName);
 754                                 if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
 755                                         /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
 756
 757                                         $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
 758                                         $nodeToAppend = $this->dom->createElement('div');
 759                                         try {
 760                                                 $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
 761                                                 $nodeToAppend->innerHTML = $siblingNode->innerHTML;
 762                                         }
 763                                         catch(Exception $e)
 764                                         {
 765                                                 $this->dbg('Could not alter siblingNode to div, reverting back to original.');
 766                                                 $nodeToAppend = $siblingNode;
 767                                                 $s--;
 768                                                 $sl--;
 769                                         }
 770                                 } else {
 771                                         $nodeToAppend = $siblingNode;
 772                                         $s--;
 773                                         $sl--;
 774                                 }
 775
 776                                 /* To ensure a node does not interfere with readability styles, remove its classnames */
 777                                 $nodeToAppend->removeAttribute('class');
 778
 779                                 /* Append sibling and subtract from our list because it removes the node when you append to another node */
 780                                 $articleContent->appendChild($nodeToAppend);
 781                         }
 782                 }
 783
 784                 /**
 785                 * So we have all of the content that we need. Now we clean it up for presentation.
 786                 **/
 787                 $this->prepArticle($articleContent);
 788
 789                 /**
 790                 * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
 791                 * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
 792                 * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
 793                 * finding the -right- content.
 794                 **/
 795                 if (strlen($this->getInnerText($articleContent, false)) < 250)
 796                 {
 797                         // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7
 798                         // in the meantime, we check and create an empty element if it's not there.
 799                         $this->reinitBody();
 800
 801                         if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
 802                                 $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
 803                                 return $this->grabArticle($this->body);
 804                         }
 805                         else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
 806                                 $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
 807                                 return $this->grabArticle($this->body);
 808                         }
 809                         else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
 810                                 $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
 811                                 return $this->grabArticle($this->body);
 812                         }
 813                         else {
 814                                 return false;
 815                         }
 816                 }
 817                 return $articleContent;
 818         }
 819
 820         /**
 821         * Remove script tags from document
 822         *
 823         * @param DOMElement
 824         * @return void
 825         */
 826         public function removeScripts($doc) {
 827                 $scripts = $doc->getElementsByTagName('script');
 828                 for($i = $scripts->length-1; $i >= 0; $i--)
 829                 {
 830                         $scripts->item($i)->parentNode->removeChild($scripts->item($i));
 831                 }
 832         }
 833
 834         /**
 835         * Get the inner text of a node.
 836         * This also strips out any excess whitespace to be found.
 837         *
 838         * @param DOMElement $
 839         * @param boolean $normalizeSpaces (default: true)
 840         * @return string
 841         **/
 842         public function getInnerText($e, $normalizeSpaces=true) {
 843                 $textContent = '';
 844
 845                 if (!isset($e->textContent) || $e->textContent == '') {
 846                         return '';
 847                 }
 848
 849                 $textContent = trim($e->textContent);
 850
 851                 if ($normalizeSpaces) {
 852                         return preg_replace($this->regexps['normalize'], ' ', $textContent);
 853                 } else {
 854                         return $textContent;
 855                 }
 856         }
 857
 858         /**
 859         * Get the number of times a string $s appears in the node $e.
 860         *
 861         * @param DOMElement $e
 862         * @param string - what to count. Default is ","
 863         * @return number (integer)
 864         **/
 865         public function getCharCount($e, $s=',') {
 866                 return substr_count($this->getInnerText($e), $s);
 867         }
 868
 869         /**
 870         * Remove the style attribute on every $e and under.
 871         *
 872         * @param DOMElement $e
 873         * @return void
 874         */
 875         public function cleanStyles($e) {
 876                 if (!is_object($e)) return;
 877                 $elems = $e->getElementsByTagName('*');
 878                 foreach ($elems as $elem) {
 879                         $elem->removeAttribute('style');
 880                 }
 881         }
 882
 883         /**
 884         * Get the density of links as a percentage of the content
 885         * This is the amount of text that is inside a link divided by the total text in the node.
 886         *
 887         * @param DOMElement $e
 888         * @return number (float)
 889         */
 890         public function getLinkDensity($e) {
 891                 $links      = $e->getElementsByTagName('a');
 892                 $textLength = strlen($this->getInnerText($e));
 893                 $linkLength = 0;
 894                 for ($i=0, $il=$links->length; $i < $il; $i++)
 895                 {
 896                         $linkLength += strlen($this->getInnerText($links->item($i)));
 897                 }
 898                 if ($textLength > 0) {
 899                         return $linkLength / $textLength;
 900                 } else {
 901                         return 0;
 902                 }
 903         }
 904
 905         /**
 906         * Get an elements class/id weight. Uses regular expressions to tell if this
 907         * element looks good or bad.
 908         *
 909         * @param DOMElement $e
 910         * @return number (Integer)
 911         */
 912         public function getClassWeight($e) {
 913                 if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
 914                         return 0;
 915                 }
 916
 917                 $weight = 0;
 918
 919                 /* Look for a special classname */
 920                 if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
 921                 {
 922                         if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
 923                                 $weight -= 25;
 924                         }
 925                         if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
 926                                 $weight += 25;
 927                         }
 928                 }
 929
 930                 /* Look for a special ID */
 931                 if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
 932                 {
 933                         if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
 934                                 $weight -= 25;
 935                         }
 936                         if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
 937                                 $weight += 25;
 938                         }
 939                 }
 940                 return $weight;
 941         }
 942
 943         /**
 944         * Remove extraneous break tags from a node.
 945         *
 946         * @param DOMElement $node
 947         * @return void
 948         */
 949         public function killBreaks($node) {
 950                 $html = $node->innerHTML;
 951                 $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
 952                 $node->innerHTML = $html;
 953         }
 954
 955         /**
 956         * Clean a node of all elements of type "tag".
 957         * (Unless it's a youtube/vimeo video. People love movies.)
 958         *
 959         * Updated 2012-09-18 to preserve youtube/vimeo iframes
 960         *
 961         * @param DOMElement $e
 962         * @param string $tag
 963         * @return void
 964         */
 965         public function clean($e, $tag) {
 966                 $targetList = $e->getElementsByTagName($tag);
 967                 $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
 968
 969                 for ($y=$targetList->length-1; $y >= 0; $y--) {
 970                         /* Allow youtube and vimeo videos through as people usually want to see those. */
 971                         if ($isEmbed) {
 972                                 $attributeValues = '';
 973                                 for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
 974                                         $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
 975                                 }
 976
 977                                 /* First, check the elements attributes to see if any of them contain youtube or vimeo */
 978                                 if (preg_match($this->regexps['video'], $attributeValues)) {
 979                                         continue;
 980                                 }
 981
 982                                 /* Then check the elements inside this element for the same. */
 983                                 if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
 984                                         continue;
 985                                 }
 986                         }
 987                         $targetList->item($y)->parentNode->removeChild($targetList->item($y));
 988                 }
 989         }
 990
 991         /**
 992         * Clean an element of all tags of type "tag" if they look fishy.
 993         * "Fishy" is an algorithm based on content length, classnames,
 994         * link density, number of images & embeds, etc.
 995         *
 996         * @param DOMElement $e
 997         * @param string $tag
 998         * @return void
 999         */
1000         public function cleanConditionally($e, $tag) {
1001                 if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
1002                         return;
1003                 }
1004
1005                 $tagsList = $e->getElementsByTagName($tag);
1006                 $curTagsLength = $tagsList->length;
1007
1008                 /**
1009                 * Gather counts for other typical elements embedded within.
1010                 * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
1011                 *
1012                 * TODO: Consider taking into account original contentScore here.
1013                 */
1014                 for ($i=$curTagsLength-1; $i >= 0; $i--) {
1015                         $weight = $this->getClassWeight($tagsList->item($i));
1016                         $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
1017
1018                         $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
1019
1020                         if ($weight + $contentScore < 0) {
1021                                 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1022                         }
1023                         else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
1024                                 /**
1025                                 * If there are not very many commas, and the number of
1026                                 * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
1027                                 **/
1028                                 $p      = $tagsList->item($i)->getElementsByTagName('p')->length;
1029                                 $img    = $tagsList->item($i)->getElementsByTagName('img')->length;
1030                                 $li     = $tagsList->item($i)->getElementsByTagName('li')->length-100;
1031                                 $input  = $tagsList->item($i)->getElementsByTagName('input')->length;
1032                                 $a              = $tagsList->item($i)->getElementsByTagName('a')->length;
1033
1034                                 $embedCount = 0;
1035                                 $embeds = $tagsList->item($i)->getElementsByTagName('embed');
1036                                 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1037                                         if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1038                                                 $embedCount++;
1039                                         }
1040                                 }
1041                                 $embeds = $tagsList->item($i)->getElementsByTagName('iframe');
1042                                 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1043                                         if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1044                                                 $embedCount++;
1045                                         }
1046                                 }
1047
1048                                 $linkDensity   = $this->getLinkDensity($tagsList->item($i));
1049                                 $contentLength = strlen($this->getInnerText($tagsList->item($i)));
1050                                 $toRemove      = false;
1051
1052                                 if ($this->lightClean) {
1053                                         $this->dbg('Light clean...');
1054                                         if ( ($img > $p) && ($img > 4) ) {
1055                                                 $this->dbg(' more than 4 images and more image elements than paragraph elements');
1056                                                 $toRemove = true;
1057                                         } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1058                                                 $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
1059                                                 $toRemove = true;
1060                                         } else if ( $input > floor($p/3) ) {
1061                                                 $this->dbg(' too many <input> elements');
1062                                                 $toRemove = true;
1063                                         } else if ($contentLength < 10 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
1064                                                 $this->dbg(' content length less than 10 chars, 0 embeds and either 0 images or more than 2 images');
1065                                                 $toRemove = true;
1066                                         } else if($weight < 25 && $linkDensity > 0.2) {
1067                                                 $this->dbg(' weight smaller than 25 and link density above 0.2');
1068                                                 $toRemove = true;
1069                                         } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
1070                                                 $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
1071                                                 $toRemove = true;
1072                                         } else if($embedCount > 3) {
1073                                                 $this->dbg(' more than 3 embeds');
1074                                                 $toRemove = true;
1075                                         }
1076                                 } else {
1077                                         $this->dbg('Standard clean...');
1078                                         if ( $img > $p ) {
1079                                                 $this->dbg(' more image elements than paragraph elements');
1080                                                 $toRemove = true;
1081                                         } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1082                                                 $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
1083                                                 $toRemove = true;
1084                                         } else if ( $input > floor($p/3) ) {
1085                                                 $this->dbg(' too many <input> elements');
1086                                                 $toRemove = true;
1087                                         } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
1088                                                 $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
1089                                                 $toRemove = true;
1090                                         } else if($weight < 25 && $linkDensity > 0.2) {
1091                                                 $this->dbg(' weight smaller than 25 and link density above 0.2');
1092                                                 $toRemove = true;
1093                                         } else if($weight >= 25 && $linkDensity > 0.5) {
1094                                                 $this->dbg(' weight above 25 but link density greater than 0.5');
1095                                                 $toRemove = true;
1096                                         } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
1097                                                 $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
1098                                                 $toRemove = true;
1099                                         }
1100                                 }
1101
1102                                 if ($toRemove) {
1103                                         //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
1104                                         $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1105                                 }
1106                         }
1107                 }
1108         }
1109
1110         /**
1111         * Clean out spurious headers from an Element. Checks things like classnames and link density.
1112         *
1113         * @param DOMElement $e
1114         * @return void
1115         */
1116         public function cleanHeaders($e) {
1117                 for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
1118                         $headers = $e->getElementsByTagName('h' . $headerIndex);
1119                         for ($i=$headers->length-1; $i >=0; $i--) {
1120                                 if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
1121                                         $headers->item($i)->parentNode->removeChild($headers->item($i));
1122                                 }
1123                         }
1124                 }
1125         }
1126
1127         public function flagIsActive($flag) {
1128                 return ($this->flags & $flag) > 0;
1129         }
1130
1131         public function addFlag($flag) {
1132                 $this->flags = $this->flags | $flag;
1133         }
1134
1135         public function removeFlag($flag) {
1136                 $this->flags = $this->flags & ~$flag;
1137         }
1138
1139         /**
1140          * Will recreate previously deleted body property
1141          *
1142          * @return void
1143          */
1144         protected function reinitBody() {
1145                 if (!isset($this->body->childNodes)) {
1146                         $this->body = $this->dom->createElement('body');
1147                         $this->body->innerHTML = $this->bodyCache;
1148                 }
1149         }
1150
1151 }
1152 ?>