3 * Arc90's Readability ported to PHP for FiveFilters.org
4 * Based on readability.js version 1.7.1 (without multi-page support)
5 * ------------------------------------------------------
6 * Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
7 * Arc90's project URL: http://lab.arc90.com/experiments/readability/
8 * JS Source: http://code.google.com/p/arc90labs-readability
9 * Ported by: Keyvan Minoukadeh, http://www.keyvan.net
10 * More information: http://fivefilters.org/content-only/
11 * License: Apache License, Version 2.0
15 * Differences between the PHP port and the original
16 * ------------------------------------------------------
17 * Arc90's Readability is designed to run in the browser. It works on the DOM
18 * tree (the parsed HTML) after the page's CSS styles have been applied and
19 * Javascript code executed. This PHP port does not run inside a browser.
20 * We use PHP's ability to parse HTML to build our DOM tree, but we cannot
21 * rely on CSS or Javascript support. As such, the results will not always
22 * match Arc90's Readability. (For example, if a web page contains CSS style
23 * rules or Javascript code which hide certain HTML elements from display,
24 * Arc90's Readability will dismiss those from consideration but our PHP port,
25 * unable to understand CSS or Javascript, will not know any better.)
27 * Another significant difference is that the aim of Arc90's Readability is
28 * to re-present the main content block of a given web page so users can
29 * read it more easily in their browsers. Correct identification, clean up,
30 * and separation of the content block is only a part of this process.
31 * This PHP port is only concerned with this part, it does not include code
32 * that relates to presentation in the browser - Arc90 already do
33 * that extremely well, and for PDF output there's FiveFilters.org's
34 * PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
36 * Finally, this class contains methods that might be useful for developers
37 * working on HTML document fragments. So without deviating too much from
38 * the original code (which I don't want to do because it makes debugging
39 * and updating more difficult), I've tried to make it a little more
40 * developer friendly. You should be able to use the methods here on
41 * existing DOMElement objects without passing an entire HTML document to
45 // This class allows us to do JavaScript like assignements to innerHTML
46 require_once(dirname(__FILE__
).'/JSLikeHTMLElement.php');
48 // Alternative usage (for testing only!)
49 // uncomment the lins below and call Readability.php in your browser
50 // passing it the URL of the page you'd like content from, e.g.:
51 // Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php
54 if (!isset($_GET['url']) || $_GET['url'] == '') {
55 die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
58 if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
59 $html = file_get_contents($url);
60 $r = new Readability($html, $url);
62 echo $r->articleContent->innerHTML;
67 public $version = '1.7.1-without-multi-page';
68 public $convertLinksToFootnotes = false;
69 public $revertForcedParagraphElements = true;
71 public $articleContent;
73 public $url = null; // optional - URL where HTML was retrieved
74 public $debug = false;
75 protected $body = null; //
76 protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
77 protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.
78 protected $success = false; // indicates whether we were able to extract or not
81 * All of the regular expressions in use within readability.
82 * Defined up here so we don't instantiate them repeatedly in loops.
84 public $regexps = array(
85 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i',
86 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
87 'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i',
88 'negative' => '/combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
89 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
90 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
91 'replaceFonts' => '/<(\/?)font[^>]*>/i',
92 // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
93 'normalize' => '/\s{2,}/',
94 'killBreaks' => '/(<br\s*\/?>(\s| ?)*){1,}/',
95 'video' => '/http:\/\/(www\.)?(youtube|vimeo)\.com/i',
96 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
100 const FLAG_STRIP_UNLIKELYS
= 1;
101 const FLAG_WEIGHT_CLASSES
= 2;
102 const FLAG_CLEAN_CONDITIONALLY
= 4;
105 * Create instance of Readability
106 * @param string UTF-8 encoded string
107 * @param string (optional) URL associated with HTML (used for footnotes)
109 function __construct($html, $url=null)
111 /* Turn all double br's into p's */
112 $html = preg_replace($this->regexps
['replaceBrs'], '</p><p>', $html);
113 $html = preg_replace($this->regexps
['replaceFonts'], '<$1span>', $html);
114 $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
115 $this->dom
= new DOMDocument();
116 $this->dom
->preserveWhiteSpace
= false;
117 $this->dom
->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
118 if (trim($html) == '') $html = '<html></html>';
119 @$this->dom
->loadHTML($html);
124 * Get article title element
127 public function getTitle() {
128 return $this->articleTitle
;
132 * Get article content element
135 public function getContent() {
136 return $this->articleContent
;
143 * 1. Prep the document by removing script tags, css, etc.
144 * 2. Build readability's DOM tree.
145 * 3. Grab the article content from the current dom tree.
146 * 4. Replace the current DOM tree with the new one.
147 * 5. Read peacefully.
149 * @return boolean true if we found content, false otherwise
151 public function init()
153 $this->removeScripts($this->dom
);
154 //die($this->getInnerHTML($this->dom->documentElement));
156 // Assume successful outcome
157 $this->success
= true;
159 $bodyElems = $this->dom
->getElementsByTagName('body');
160 if ($bodyElems->length
> 0) {
161 if ($this->bodyCache
== null) {
162 $this->bodyCache
= $bodyElems->item(0)->innerHTML
;
164 if ($this->body
== null) {
165 $this->body
= $bodyElems->item(0);
169 $this->prepDocument();
171 //die($this->dom->documentElement->parentNode->nodeType);
172 //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));
173 //die($this->getInnerHTML($this->dom->documentElement));
175 /* Build readability's DOM tree */
176 $overlay = $this->dom
->createElement('div');
177 $innerDiv = $this->dom
->createElement('div');
178 $articleTitle = $this->getArticleTitle();
179 $articleContent = $this->grabArticle();
181 if (!$articleContent) {
182 $this->success
= false;
183 $articleContent = $this->dom
->createElement('div');
184 $articleContent->setAttribute('id', 'readability-content');
185 $articleContent->innerHTML
= '<p>Sorry, Readability was unable to parse this page for content.</p>';
188 $overlay->setAttribute('id', 'readOverlay');
189 $innerDiv->setAttribute('id', 'readInner');
191 /* Glue the structure of our document together. */
192 $innerDiv->appendChild($articleTitle);
193 $innerDiv->appendChild($articleContent);
194 $overlay->appendChild($innerDiv);
196 /* Clear the old HTML, insert the new content. */
197 $this->body
->innerHTML
= '';
198 $this->body
->appendChild($overlay);
199 //document.body.insertBefore(overlay, document.body.firstChild);
200 $this->body
->removeAttribute('style');
202 $this->postProcessContent($articleContent);
204 // Set title and content instance variables
205 $this->articleTitle
= $articleTitle;
206 $this->articleContent
= $articleContent;
208 return $this->success
;
214 protected function dbg($msg) {
215 if ($this->debug
) echo '* ',$msg, '<br />', "\n";
219 * Run any post-process modifications to article content as necessary.
224 public function postProcessContent($articleContent) {
225 if ($this->convertLinksToFootnotes
&& !preg_match('/wikipedia\.org/', @$this->url
)) {
226 $this->addFootnotes($articleContent);
231 * Get the article title as an H1.
235 protected function getArticleTitle() {
240 $curTitle = $origTitle = $this->getInnerText($this->dom
->getElementsByTagName('title')->item(0));
241 } catch(Exception
$e) {}
243 if (preg_match('/ [\|\-] /', $curTitle))
245 $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
247 if (count(explode(' ', $curTitle)) < 3) {
248 $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);
251 else if (strpos($curTitle, ': ') !== false)
253 $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);
255 if (count(explode(' ', $curTitle)) < 3) {
256 $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);
259 else if(strlen($curTitle) > 150 ||
strlen($curTitle) < 15)
261 $hOnes = $this->dom
->getElementsByTagName('h1');
262 if($hOnes->length
== 1)
264 $curTitle = $this->getInnerText($hOnes->item(0));
268 $curTitle = trim($curTitle);
270 if (count(explode(' ', $curTitle)) <= 4) {
271 $curTitle = $origTitle;
274 $articleTitle = $this->dom
->createElement('h1');
275 $articleTitle->innerHTML
= $curTitle;
277 return $articleTitle;
281 * Prepare the HTML document for readability to scrape it.
282 * This includes things like stripping javascript, CSS, and handling terrible markup.
286 protected function prepDocument() {
288 * In some cases a body element can't be found (if the HTML is totally hosed for example)
289 * so we create a new body node and append it to the document.
291 if ($this->body
== null)
293 $this->body
= $this->dom
->createElement('body');
294 $this->dom
->documentElement
->appendChild($this->body
);
297 $this->body
->setAttribute('id', 'readabilityBody');
299 /* Remove all style tags in head */
300 $styleTags = $this->dom
->getElementsByTagName('style');
301 for ($i = $styleTags->length
-1; $i >= 0; $i--)
303 $styleTags->item($i)->parentNode
->removeChild($styleTags->item($i));
306 /* Turn all double br's into p's */
307 /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
308 //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');
309 // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.
310 // Manipulating innerHTML as it's done in JS is not possible in PHP.
314 * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
315 * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
319 public function addFootnotes($articleContent) {
320 $footnotesWrapper = $this->dom
->createElement('div');
321 $footnotesWrapper->setAttribute('id', 'readability-footnotes');
322 $footnotesWrapper->innerHTML
= '<h3>References</h3>';
324 $articleFootnotes = $this->dom
->createElement('ol');
325 $articleFootnotes->setAttribute('id', 'readability-footnotes-list');
326 $footnotesWrapper->appendChild($articleFootnotes);
328 $articleLinks = $articleContent->getElementsByTagName('a');
331 for ($i = 0; $i < $articleLinks->length
; $i++
)
333 $articleLink = $articleLinks->item($i);
334 $footnoteLink = $articleLink->cloneNode(true);
335 $refLink = $this->dom
->createElement('a');
336 $footnote = $this->dom
->createElement('li');
337 $linkDomain = @parse_url
($footnoteLink->getAttribute('href'), PHP_URL_HOST
);
338 if (!$linkDomain && isset($this->url
)) $linkDomain = @parse_url
($this->url
, PHP_URL_HOST
);
339 //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host,
340 $linkText = $this->getInnerText($articleLink);
342 if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) ||
preg_match($this->regexps
['skipFootnoteLink'], $linkText)) {
348 /** Add a superscript reference after the article link */
349 $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);
350 $refLink->innerHTML
= '<small><sup>[' . $linkCount . ']</sup></small>';
351 $refLink->setAttribute('class', 'readability-DoNotFootnote');
352 $refLink->setAttribute('style', 'color: inherit;');
354 //TODO: does this work or should we use DOMNode.isSameNode()?
355 if ($articleLink->parentNode
->lastChild
== $articleLink) {
356 $articleLink->parentNode
->appendChild($refLink);
358 $articleLink->parentNode
->insertBefore($refLink, $articleLink->nextSibling
);
361 $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
362 $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
364 $footnote->innerHTML
= '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';
366 $footnoteLink->innerHTML
= ($footnoteLink->getAttribute('title') != '' ?
$footnoteLink->getAttribute('title') : $linkText);
367 $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
369 $footnote->appendChild($footnoteLink);
370 if ($linkDomain) $footnote->innerHTML
= $footnote->innerHTML
. '<small> (' . $linkDomain . ')</small>';
372 $articleFootnotes->appendChild($footnote);
375 if ($linkCount > 0) {
376 $articleContent->appendChild($footnotesWrapper);
381 * Reverts P elements with class 'readability-styled'
382 * to text nodes - which is what they were before.
387 function revertReadabilityStyledElements($articleContent) {
388 $xpath = new DOMXPath($articleContent->ownerDocument
);
389 $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);
390 //$elems = $articleContent->getElementsByTagName('p');
391 for ($i = $elems->length
-1; $i >= 0; $i--) {
392 $e = $elems->item($i);
393 $e->parentNode
->replaceChild($articleContent->ownerDocument
->createTextNode($e->textContent
), $e);
394 //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {
395 // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);
401 * Prepare the article node for display. Clean out any inline styles,
402 * iframes, forms, strip extraneous <p> tags, etc.
407 function prepArticle($articleContent) {
408 $this->cleanStyles($articleContent);
409 $this->killBreaks($articleContent);
410 if ($this->revertForcedParagraphElements
) {
411 $this->revertReadabilityStyledElements($articleContent);
414 /* Clean out junk from the article content */
415 $this->cleanConditionally($articleContent, 'form');
416 $this->clean($articleContent, 'object');
417 $this->clean($articleContent, 'h1');
420 * If there is only one h2, they are probably using it
421 * as a header and not a subheader, so remove it since we already have a header.
423 if ($articleContent->getElementsByTagName('h2')->length
== 1) {
424 $this->clean($articleContent, 'h2');
426 $this->clean($articleContent, 'iframe');
428 $this->cleanHeaders($articleContent);
430 /* Do these last as the previous stuff may have removed junk that will affect these */
431 $this->cleanConditionally($articleContent, 'table');
432 $this->cleanConditionally($articleContent, 'ul');
433 $this->cleanConditionally($articleContent, 'div');
435 /* Remove extra paragraphs */
436 $articleParagraphs = $articleContent->getElementsByTagName('p');
437 for ($i = $articleParagraphs->length
-1; $i >= 0; $i--)
439 $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length
;
440 $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length
;
441 $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length
;
443 if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
445 $articleParagraphs->item($i)->parentNode
->removeChild($articleParagraphs->item($i));
450 $articleContent->innerHTML
= preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML
);
451 //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');
453 catch (Exception
$e) {
454 $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
459 * Initialize a node with the readability object. Also checks the
460 * className/id for special names to add to its score.
465 protected function initializeNode($node) {
466 $readability = $this->dom
->createAttribute('readability');
467 $readability->value
= 0; // this is our contentScore
468 $node->setAttributeNode($readability);
470 switch (strtoupper($node->tagName
)) { // unsure if strtoupper is needed, but using it just in case
472 $readability->value +
= 5;
478 $readability->value +
= 3;
489 $readability->value
-= 3;
499 $readability->value
-= 5;
502 $readability->value +
= $this->getClassWeight($node);
506 * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
507 * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
511 protected function grabArticle($page=null) {
512 $stripUnlikelyCandidates = $this->flagIsActive(self
::FLAG_STRIP_UNLIKELYS
);
513 if (!$page) $page = $this->dom
;
514 $allElements = $page->getElementsByTagName('*');
516 * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
517 * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
519 * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
520 * TODO: Shouldn't this be a reverse traversal?
523 $nodesToScore = array();
524 for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++
) {
525 //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
526 //$node = $targetList->item($nodeIndex);
527 $tagName = strtoupper($node->tagName
);
528 /* Remove unlikely candidates */
529 if ($stripUnlikelyCandidates) {
530 $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
532 preg_match($this->regexps
['unlikelyCandidates'], $unlikelyMatchString) &&
533 !preg_match($this->regexps
['okMaybeItsACandidate'], $unlikelyMatchString) &&
537 $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
538 //$nodesToRemove[] = $node;
539 $node->parentNode
->removeChild($node);
545 if ($tagName == 'P' ||
$tagName == 'TD' ||
$tagName == 'PRE') {
546 $nodesToScore[] = $node;
549 /* Turn all divs that don't have children block level elements into p's */
550 if ($tagName == 'DIV') {
551 if (!preg_match($this->regexps
['divToPElements'], $node->innerHTML
)) {
552 //$this->dbg('Altering div to p');
553 $newNode = $this->dom
->createElement('p');
555 $newNode->innerHTML
= $node->innerHTML
;
556 //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
557 $node->parentNode
->replaceChild($newNode, $node);
559 $nodesToScore[] = $node; // or $newNode?
561 catch(Exception
$e) {
562 $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
568 // TODO: change these p elements back to text nodes after processing
569 for ($i = 0, $il = $node->childNodes
->length
; $i < $il; $i++
) {
570 $childNode = $node->childNodes
->item($i);
571 if ($childNode->nodeType
== 3) { // XML_TEXT_NODE
572 //$this->dbg('replacing text node with a p tag with the same content.');
573 $p = $this->dom
->createElement('p');
574 $p->innerHTML
= $childNode->nodeValue
;
575 $p->setAttribute('style', 'display: inline;');
576 $p->setAttribute('class', 'readability-styled');
577 $childNode->parentNode
->replaceChild($p, $childNode);
585 * Loop through all paragraphs, and assign a score to them based on how content-y they look.
586 * Then add their score to their parent node.
588 * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
590 $candidates = array();
591 for ($pt=0; $pt < count($nodesToScore); $pt++
) {
592 $parentNode = $nodesToScore[$pt]->parentNode
;
593 // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
594 $grandParentNode = !$parentNode ?
null : (($parentNode->parentNode
instanceof DOMElement
) ?
$parentNode->parentNode
: null);
595 $innerText = $this->getInnerText($nodesToScore[$pt]);
597 if (!$parentNode ||
!isset($parentNode->tagName
)) {
601 /* If this paragraph is less than 25 characters, don't even count it. */
602 if(strlen($innerText) < 25) {
606 /* Initialize readability data for the parent. */
607 if (!$parentNode->hasAttribute('readability'))
609 $this->initializeNode($parentNode);
610 $candidates[] = $parentNode;
613 /* Initialize readability data for the grandparent. */
614 if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName
))
616 $this->initializeNode($grandParentNode);
617 $candidates[] = $grandParentNode;
622 /* Add a point for the paragraph itself as a base. */
625 /* Add points for any commas within this paragraph */
626 $contentScore +
= count(explode(',', $innerText));
628 /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
629 $contentScore +
= min(floor(strlen($innerText) / 100), 3);
631 /* Add the score to the parent. The grandparent gets half. */
632 $parentNode->getAttributeNode('readability')->value +
= $contentScore;
634 if ($grandParentNode) {
635 $grandParentNode->getAttributeNode('readability')->value +
= $contentScore/2;
640 * After we've calculated scores, loop through all of the possible candidate nodes we found
641 * and find the one with the highest score.
643 $topCandidate = null;
644 for ($c=0, $cl=count($candidates); $c < $cl; $c++
)
647 * Scale the final candidates score based on link density. Good content should have a
648 * relatively small link density (5% or less) and be mostly unaffected by this operation.
650 $readability = $candidates[$c]->getAttributeNode('readability');
651 $readability->value
= $readability->value
* (1-$this->getLinkDensity($candidates[$c]));
653 $this->dbg('Candidate: ' . $candidates[$c]->tagName
. ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value
);
655 if (!$topCandidate ||
$readability->value
> (int)$topCandidate->getAttribute('readability')) {
656 $topCandidate = $candidates[$c];
661 * If we still have no top candidate, just use the body as a last resort.
662 * We also have to copy the body node so it is something we can modify.
664 if ($topCandidate === null ||
strtoupper($topCandidate->tagName
) == 'BODY')
666 $topCandidate = $this->dom
->createElement('div');
667 $topCandidate->innerHTML
= ($page instanceof DOMDocument
) ?
$page->saveXML($page->documentElement
) : $page->innerHTML
;
668 $page->innerHTML
= '';
669 $page->appendChild($topCandidate);
670 $this->initializeNode($topCandidate);
674 * Now that we have the top candidate, look through its siblings for content that might also be related.
675 * Things like preambles, content split by ads that we removed, etc.
677 $articleContent = $this->dom
->createElement('div');
678 $articleContent->setAttribute('id', 'readability-content');
679 $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
680 $siblingNodes = $topCandidate->parentNode
->childNodes
;
682 for ($s=0, $sl=$siblingNodes->length
; $s < $sl; $s++
)
684 $siblingNode = $siblingNodes->item($s);
687 $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName
. (($siblingNode->nodeType
=== XML_ELEMENT_NODE
&& $siblingNode->hasAttribute('readability')) ?
(' with score ' . $siblingNode->getAttribute('readability')) : ''));
689 //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
691 if ($siblingNode === $topCandidate)
692 // or if ($siblingNode->isSameNode($topCandidate))
698 /* Give a bonus if sibling nodes and top candidates have the example same classname */
699 if ($siblingNode->nodeType
=== XML_ELEMENT_NODE
&& $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
700 $contentBonus +
= ((int)$topCandidate->getAttribute('readability')) * 0.2;
703 if ($siblingNode->nodeType
=== XML_ELEMENT_NODE
&& $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) +
$contentBonus) >= $siblingScoreThreshold)
708 if (strtoupper($siblingNode->nodeName
) == 'P') {
709 $linkDensity = $this->getLinkDensity($siblingNode);
710 $nodeContent = $this->getInnerText($siblingNode);
711 $nodeLength = strlen($nodeContent);
713 if ($nodeLength > 80 && $linkDensity < 0.25)
717 else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
725 $this->dbg('Appending node: ' . $siblingNode->nodeName
);
727 $nodeToAppend = null;
728 $sibNodeName = strtoupper($siblingNode->nodeName
);
729 if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
730 /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
732 $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
733 $nodeToAppend = $this->dom
->createElement('div');
735 $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
736 $nodeToAppend->innerHTML
= $siblingNode->innerHTML
;
740 $this->dbg('Could not alter siblingNode to div, reverting back to original.');
741 $nodeToAppend = $siblingNode;
746 $nodeToAppend = $siblingNode;
751 /* To ensure a node does not interfere with readability styles, remove its classnames */
752 $nodeToAppend->removeAttribute('class');
754 /* Append sibling and subtract from our list because it removes the node when you append to another node */
755 $articleContent->appendChild($nodeToAppend);
760 * So we have all of the content that we need. Now we clean it up for presentation.
762 $this->prepArticle($articleContent);
765 * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
766 * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
767 * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
768 * finding the -right- content.
770 if (strlen($this->getInnerText($articleContent, false)) < 250)
772 $this->body
->innerHTML
= $this->bodyCache
;
774 if ($this->flagIsActive(self
::FLAG_STRIP_UNLIKELYS
)) {
775 $this->removeFlag(self
::FLAG_STRIP_UNLIKELYS
);
776 return $this->grabArticle($this->body
);
778 else if ($this->flagIsActive(self
::FLAG_WEIGHT_CLASSES
)) {
779 $this->removeFlag(self
::FLAG_WEIGHT_CLASSES
);
780 return $this->grabArticle($this->body
);
782 else if ($this->flagIsActive(self
::FLAG_CLEAN_CONDITIONALLY
)) {
783 $this->removeFlag(self
::FLAG_CLEAN_CONDITIONALLY
);
784 return $this->grabArticle($this->body
);
790 return $articleContent;
794 * Remove script tags from document
799 public function removeScripts($doc) {
800 $scripts = $doc->getElementsByTagName('script');
801 for($i = $scripts->length
-1; $i >= 0; $i--)
803 $scripts->item($i)->parentNode
->removeChild($scripts->item($i));
808 * Get the inner text of a node.
809 * This also strips out any excess whitespace to be found.
811 * @param DOMElement $
812 * @param boolean $normalizeSpaces (default: true)
815 public function getInnerText($e, $normalizeSpaces=true) {
818 if (!isset($e->textContent
) ||
$e->textContent
== '') {
822 $textContent = trim($e->textContent
);
824 if ($normalizeSpaces) {
825 return preg_replace($this->regexps
['normalize'], ' ', $textContent);
832 * Get the number of times a string $s appears in the node $e.
834 * @param DOMElement $e
835 * @param string - what to count. Default is ","
836 * @return number (integer)
838 public function getCharCount($e, $s=',') {
839 return substr_count($this->getInnerText($e), $s);
843 * Remove the style attribute on every $e and under.
845 * @param DOMElement $e
848 public function cleanStyles($e) {
849 $elems = $e->getElementsByTagName('*');
850 foreach ($elems as $elem) {
851 $elem->removeAttribute('style');
856 * Get the density of links as a percentage of the content
857 * This is the amount of text that is inside a link divided by the total text in the node.
859 * @param DOMElement $e
860 * @return number (float)
862 public function getLinkDensity($e) {
863 $links = $e->getElementsByTagName('a');
864 $textLength = strlen($this->getInnerText($e));
866 for ($i=0, $il=$links->length
; $i < $il; $i++
)
868 $linkLength +
= strlen($this->getInnerText($links->item($i)));
870 if ($textLength > 0) {
871 return $linkLength / $textLength;
878 * Get an elements class/id weight. Uses regular expressions to tell if this
879 * element looks good or bad.
881 * @param DOMElement $e
882 * @return number (Integer)
884 public function getClassWeight($e) {
885 if(!$this->flagIsActive(self
::FLAG_WEIGHT_CLASSES
)) {
891 /* Look for a special classname */
892 if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
894 if (preg_match($this->regexps
['negative'], $e->getAttribute('class'))) {
897 if (preg_match($this->regexps
['positive'], $e->getAttribute('class'))) {
902 /* Look for a special ID */
903 if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
905 if (preg_match($this->regexps
['negative'], $e->getAttribute('id'))) {
908 if (preg_match($this->regexps
['positive'], $e->getAttribute('id'))) {
916 * Remove extraneous break tags from a node.
918 * @param DOMElement $node
921 public function killBreaks($node) {
922 $html = $node->innerHTML
;
923 $html = preg_replace($this->regexps
['killBreaks'], '<br />', $html);
924 $node->innerHTML
= $html;
928 * Clean a node of all elements of type "tag".
929 * (Unless it's a youtube/vimeo video. People love movies.)
931 * @param DOMElement $e
935 public function clean($e, $tag) {
936 $targetList = $e->getElementsByTagName($tag);
937 $isEmbed = ($tag == 'object' ||
$tag == 'embed');
939 for ($y=$targetList->length
-1; $y >= 0; $y--) {
940 /* Allow youtube and vimeo videos through as people usually want to see those. */
942 $attributeValues = '';
943 for ($i=0, $il=$targetList->item($y)->attributes
->length
; $i < $il; $i++
) {
944 $attributeValues .= $targetList->item($y)->attributes
->item($i)->value
. '|'; // DOMAttr? (TODO: test)
947 /* First, check the elements attributes to see if any of them contain youtube or vimeo */
948 if (preg_match($this->regexps
['video'], $attributeValues)) {
952 /* Then check the elements inside this element for the same. */
953 if (preg_match($this->regexps
['video'], $targetList->item($y)->innerHTML
)) {
957 $targetList->item($y)->parentNode
->removeChild($targetList->item($y));
962 * Clean an element of all tags of type "tag" if they look fishy.
963 * "Fishy" is an algorithm based on content length, classnames,
964 * link density, number of images & embeds, etc.
966 * @param DOMElement $e
970 public function cleanConditionally($e, $tag) {
971 if (!$this->flagIsActive(self
::FLAG_CLEAN_CONDITIONALLY
)) {
975 $tagsList = $e->getElementsByTagName($tag);
976 $curTagsLength = $tagsList->length
;
979 * Gather counts for other typical elements embedded within.
980 * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
982 * TODO: Consider taking into account original contentScore here.
984 for ($i=$curTagsLength-1; $i >= 0; $i--) {
985 $weight = $this->getClassWeight($tagsList->item($i));
986 $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ?
(int)$tagsList->item($i)->getAttribute('readability') : 0;
988 $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName
. ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ?
(' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
990 if ($weight +
$contentScore < 0) {
991 $tagsList->item($i)->parentNode
->removeChild($tagsList->item($i));
993 else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
995 * If there are not very many commas, and the number of
996 * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
998 $p = $tagsList->item($i)->getElementsByTagName('p')->length
;
999 $img = $tagsList->item($i)->getElementsByTagName('img')->length
;
1000 $li = $tagsList->item($i)->getElementsByTagName('li')->length
-100;
1001 $input = $tagsList->item($i)->getElementsByTagName('input')->length
;
1004 $embeds = $tagsList->item($i)->getElementsByTagName('embed');
1005 for ($ei=0, $il=$embeds->length
; $ei < $il; $ei++
) {
1006 if (preg_match($this->regexps
['video'], $embeds->item($ei)->getAttribute('src'))) {
1011 $linkDensity = $this->getLinkDensity($tagsList->item($i));
1012 $contentLength = strlen($this->getInnerText($tagsList->item($i)));
1017 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1019 } else if ( $input > floor($p/3) ) {
1021 } else if ($contentLength < 25 && ($img === 0 ||
$img > 2) ) {
1023 } else if($weight < 25 && $linkDensity > 0.2) {
1025 } else if($weight >= 25 && $linkDensity > 0.5) {
1027 } else if(($embedCount == 1 && $contentLength < 75) ||
$embedCount > 1) {
1032 $tagsList->item($i)->parentNode
->removeChild($tagsList->item($i));
1039 * Clean out spurious headers from an Element. Checks things like classnames and link density.
1041 * @param DOMElement $e
1044 public function cleanHeaders($e) {
1045 for ($headerIndex = 1; $headerIndex < 3; $headerIndex++
) {
1046 $headers = $e->getElementsByTagName('h' . $headerIndex);
1047 for ($i=$headers->length
-1; $i >=0; $i--) {
1048 if ($this->getClassWeight($headers->item($i)) < 0 ||
$this->getLinkDensity($headers->item($i)) > 0.33) {
1049 $headers->item($i)->parentNode
->removeChild($headers->item($i));
1055 public function flagIsActive($flag) {
1056 return ($this->flags
& $flag) > 0;
1059 public function addFlag($flag) {
1060 $this->flags
= $this->flags |
$flag;
1063 public function removeFlag($flag) {
1064 $this->flags
= $this->flags
& ~
$flag;