X-Git-Url: https://git.wh0rd.org/?a=blobdiff_plain;f=include%2Frssfuncs.php;h=a64d60e1eebbbc84ba29e89dcda4b7713cb035c1;hb=e2cf81e21406f0b507446cf369f7da172ce9da14;hp=d4b6e26eab78751ec9cb63632530e13c4b512224;hpb=aea03558bb7c0ac0015a1f1e41c5f5eb1722132e;p=tt-rss.git diff --git a/include/rssfuncs.php b/include/rssfuncs.php index d4b6e26e..a64d60e1 100644 --- a/include/rssfuncs.php +++ b/include/rssfuncs.php @@ -1,7 +1,7 @@ time() - $cache_age) { - if ($debug_enabled) { - _debug("update_rss_feed: using local cache."); - } + _debug("using local cache.", $debug_enabled); if ($cache_timestamp > $last_updated_timestamp) { @$rss_data = file_get_contents($cache_filename); @@ -285,40 +271,60 @@ @$rss = unserialize($rss_data); } } else if (!$force_refetch) { - if ($debug_enabled) { - _debug("update_rss_feed: local cache valid and older than last_updated, nothing to do."); - } + _debug("local cache valid and older than last_updated, nothing to do.", $debug_enabled); return; } + } else { + _debug("local cache will not be used for this feed", $debug_enabled); + $cache_timestamp = 0; } if (!$rss) { if (!$feed_data) { - if ($debug_enabled) { - _debug("update_rss_feed: fetching [$fetch_url] (ts: $cache_timestamp/$last_updated_timestamp)"); - } + _debug("fetching [$fetch_url] (ts: $cache_timestamp/$last_updated_timestamp)", $debug_enabled); $feed_data = fetch_file_contents($fetch_url, false, $auth_login, $auth_pass, false, $no_cache ? FEED_FETCH_NO_CACHE_TIMEOUT : FEED_FETCH_TIMEOUT, $force_refetch ? 0 : max($last_updated_timestamp, $cache_timestamp)); - if ($debug_enabled) { - _debug("update_rss_feed: fetch done."); + global $fetch_curl_used; + + if (!$fetch_curl_used) { + $tmp = @gzdecode($feed_data); + + if ($tmp) $feed_data = $tmp; } - $feed_data = gzdecode($feed_data); + $feed_data = trim($feed_data); + + _debug("fetch done.", $debug_enabled); + if ($feed_data) { + $error = verify_feed_xml($feed_data); + + if ($error) { + _debug("error verifying XML, code: " . $error->code, $debug_enabled); + + if ($error->code == 26) { + _debug("got error 26, trying to decode entities...", $debug_enabled); + + $feed_data = html_entity_decode($feed_data, ENT_COMPAT, 'UTF-8'); + + $error = verify_feed_xml($feed_data); + + if ($error) $feed_data = ''; + } + } + } } if (!$feed_data) { global $fetch_last_error; global $fetch_last_error_code; - if ($debug_enabled) { - _debug("update_rss_feed: unable to fetch: $fetch_last_error [$fetch_last_error_code]"); - } + _debug("unable to fetch: $fetch_last_error [$fetch_last_error_code]", $debug_enabled); $error_escaped = ''; @@ -326,9 +332,7 @@ if ($fetch_last_error_code != 304) { $error_escaped = db_escape_string($fetch_last_error); } else { - if ($debug_enabled) { - _debug("update_rss_feed: source claims data not modified, nothing to do."); - } + _debug("source claims data not modified, nothing to do.", $debug_enabled); } db_query( @@ -339,15 +343,15 @@ } } - $pluginhost = new PluginHost(Db::get()); - $pluginhost->set_debug($debug_enabled); + $pluginhost = new PluginHost(); + $pluginhost->set_debug($debug_enabled, $debug_enabled); $user_plugins = get_pref("_ENABLED_PLUGINS", $owner_uid); - $pluginhost->load(PLUGINS, $pluginhost::KIND_ALL); - $pluginhost->load($user_plugins, $pluginhost::KIND_USER, $owner_uid); + $pluginhost->load(PLUGINS, PluginHost::KIND_ALL); + $pluginhost->load($user_plugins, PluginHost::KIND_USER, $owner_uid); $pluginhost->load_data(); - foreach ($pluginhost->get_hooks($pluginhost::HOOK_FEED_FETCHED) as $plugin) { + foreach ($pluginhost->get_hooks(PluginHost::HOOK_FEED_FETCHED) as $plugin) { $feed_data = $plugin->hook_feed_fetched($feed_data); } @@ -376,19 +380,15 @@ $new_rss_hash = sha1($rss_data); if ($new_rss_hash != $rss_hash) { - if ($debug_enabled) { - _debug("update_rss_feed: saving $cache_filename"); - } + _debug("saving $cache_filename", $debug_enabled); @file_put_contents($cache_filename, serialize($rss)); } } // We use local pluginhost here because we need to load different per-user feed plugins - $pluginhost->run_hooks($pluginhost::HOOK_FEED_PARSED, "hook_feed_parsed", $rss); + $pluginhost->run_hooks(PluginHost::HOOK_FEED_PARSED, "hook_feed_parsed", $rss); - if ($debug_enabled) { - _debug("update_rss_feed: processing feed data..."); - } + _debug("processing feed data...", $debug_enabled); // db_query("BEGIN"); @@ -398,7 +398,7 @@ $favicon_interval_qpart = "favicon_last_checked < DATE_SUB(NOW(), INTERVAL 12 HOUR)"; } - $result = db_query("SELECT title,site_url,owner_uid, + $result = db_query("SELECT title,site_url,owner_uid,favicon_avg_color, (favicon_last_checked IS NULL OR $favicon_interval_qpart) AS favicon_needs_check FROM ttrss_feeds WHERE id = '$feed'"); @@ -407,26 +407,40 @@ $orig_site_url = db_fetch_result($result, 0, "site_url"); $favicon_needs_check = sql_bool_to_bool(db_fetch_result($result, 0, "favicon_needs_check")); + $favicon_avg_color = db_fetch_result($result, 0, "favicon_avg_color"); $owner_uid = db_fetch_result($result, 0, "owner_uid"); $site_url = db_escape_string(mb_substr(rewrite_relative_url($fetch_url, $rss->get_link()), 0, 245)); if ($favicon_needs_check || $force_refetch) { - if ($debug_enabled) { - _debug("update_rss_feed: checking favicon..."); - } - check_feed_favicon($site_url, $feed, $link); + /* terrible hack: if we crash on floicon shit here, we won't check + * the icon avgcolor again (unless the icon got updated) */ + + db_query("UPDATE ttrss_feeds SET favicon_avg_color = 'fail' WHERE + id = '$feed'"); + $favicon_file = ICONS_DIR . "/$feed.ico"; + $favicon_modified = @filemtime($favicon_file); + + _debug("checking favicon...", $debug_enabled); - if (file_exists($favicon_file)) { + check_feed_favicon($site_url, $feed, $link); + $favicon_modified_new = @filemtime($favicon_file); + + if ($favicon_modified_new > $favicon_modified) + $favicon_avg_color = ''; + + if (file_exists($favicon_file) && function_exists("imagecreatefromstring") && $favicon_avg_color == '') { require_once "colors.php"; $favicon_color = db_escape_string( calculate_avg_color($favicon_file)); $favicon_colorstring = ",favicon_avg_color = '".$favicon_color."'"; + } else if ($favicon_avg_color == 'fail') { + _debug("floicon failed on this file, not trying to recalculate avg color"); } db_query("UPDATE ttrss_feeds SET favicon_last_checked = NOW() @@ -438,9 +452,7 @@ $feed_title = db_escape_string($rss->get_title()); - if ($debug_enabled) { - _debug("update_rss_feed: registering title: $feed_title"); - } + _debug("registering title: $feed_title", $debug_enabled); db_query("UPDATE ttrss_feeds SET title = '$feed_title' WHERE id = '$feed'"); @@ -451,24 +463,17 @@ site_url = '$site_url' WHERE id = '$feed'"); } - if ($debug_enabled) { - _debug("update_rss_feed: loading filters & labels..."); - } + _debug("loading filters & labels...", $debug_enabled); $filters = load_filters($feed, $owner_uid); $labels = get_all_labels($owner_uid); - if ($debug_enabled) { - //print_r($filters); - _debug("update_rss_feed: " . count($filters) . " filters loaded."); - } + _debug("" . count($filters) . " filters loaded.", $debug_enabled); $items = $rss->get_items(); if (!is_array($items)) { - if ($debug_enabled) { - _debug("update_rss_feed: no articles found."); - } + _debug("no articles found.", $debug_enabled); db_query("UPDATE ttrss_feeds SET last_updated = NOW(), last_error = '' WHERE id = '$feed'"); @@ -478,7 +483,7 @@ if ($pubsub_state != 2 && PUBSUBHUBBUB_ENABLED) { - if ($debug_enabled) _debug("update_rss_feed: checking for PUSH hub..."); + _debug("checking for PUSH hub...", $debug_enabled); $feed_hub_url = false; @@ -491,7 +496,7 @@ } } - if ($debug_enabled) _debug("update_rss_feed: feed hub url: $feed_hub_url"); + _debug("feed hub url: $feed_hub_url", $debug_enabled); if ($feed_hub_url && function_exists('curl_init') && !ini_get("open_basedir")) { @@ -505,17 +510,14 @@ $rc = $s->subscribe($fetch_url); - if ($debug_enabled) - _debug("update_rss_feed: feed hub url found, subscribe request sent."); + _debug("feed hub url found, subscribe request sent.", $debug_enabled); db_query("UPDATE ttrss_feeds SET pubsub_state = 1 WHERE id = '$feed'"); } } - if ($debug_enabled) { - _debug("update_rss_feed: processing articles..."); - } + _debug("processing articles...", $debug_enabled); foreach ($items as $item) { if ($_REQUEST['xdebug'] == 3) { @@ -532,9 +534,7 @@ $entry_guid_hashed = db_escape_string('SHA1:' . sha1($entry_guid)); - if ($debug_enabled) { - _debug("update_rss_feed: guid $entry_guid / $entry_guid_hashed"); - } + _debug("guid $entry_guid / $entry_guid_hashed", $debug_enabled); $entry_timestamp = ""; @@ -549,18 +549,14 @@ $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); - if ($debug_enabled) { - _debug("update_rss_feed: date $entry_timestamp [$entry_timestamp_fmt]"); - } + _debug("date $entry_timestamp [$entry_timestamp_fmt]", $debug_enabled); - $entry_title = html_entity_decode($item->get_title()); + $entry_title = html_entity_decode($item->get_title(), ENT_COMPAT, 'UTF-8'); $entry_link = rewrite_relative_url($site_url, $item->get_link()); - if ($debug_enabled) { - _debug("update_rss_feed: title $entry_title"); - _debug("update_rss_feed: link $entry_link"); - } + _debug("title $entry_title", $debug_enabled); + _debug("link $entry_link", $debug_enabled); if (!$entry_title) $entry_title = date("Y-m-d H:i:s", $entry_timestamp);; @@ -568,7 +564,7 @@ if (!$entry_content) $entry_content = $item->get_description(); if ($_REQUEST["xdebug"] == 2) { - print "update_rss_feed: content: "; + print "content: "; print $entry_content; print "\n"; } @@ -579,14 +575,12 @@ $entry_author_item = $item->get_author(); $entry_author = $entry_author_item->get_name(); if (!$entry_author) $entry_author = $entry_author_item->get_email(); - - $entry_author = db_escape_string($entry_author); } $entry_guid = db_escape_string(mb_substr($entry_guid, 0, 245)); - $entry_comments = db_escape_string(mb_substr($entry_comments, 0, 245)); - $entry_author = db_escape_string(mb_substr($entry_author, 0, 245)); + $entry_comments = db_escape_string(mb_substr(trim($entry_comments), 0, 245)); + $entry_author = db_escape_string(mb_substr(trim($entry_author), 0, 245)); $num_comments = $item->get_item_tags('http://purl.org/rss/1.0/modules/slash/', 'comments'); @@ -596,11 +590,9 @@ $num_comments = 0; } - if ($debug_enabled) { - _debug("update_rss_feed: author $entry_author"); - _debug("update_rss_feed: num_comments: $num_comments"); - _debug("update_rss_feed: looking for tags [1]..."); - } + _debug("author $entry_author", $debug_enabled); + _debug("num_comments: $num_comments", $debug_enabled); + _debug("looking for tags [1]...", $debug_enabled); // parse entries into tags @@ -614,34 +606,19 @@ } } - if ($debug_enabled) { - _debug("update_rss_feed: category tags:"); - print_r($additional_tags); - } - - if ($debug_enabled) { - _debug("update_rss_feed: looking for tags [2]..."); - } + _debug("category tags:", $debug_enabled); + _debug("looking for tags [2]...", $debug_enabled); $entry_tags = array_unique($additional_tags); for ($i = 0; $i < count($entry_tags); $i++) $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8'); - if ($debug_enabled) { - //_debug("update_rss_feed: unfiltered tags found:"); - //print_r($entry_tags); - } - - if ($debug_enabled) { - _debug("update_rss_feed: done collecting data."); - } + _debug("done collecting data.", $debug_enabled); // TODO: less memory-hungry implementation - if ($debug_enabled) { - _debug("update_rss_feed: applying plugin filters.."); - } + _debug("applying plugin filters..", $debug_enabled); // FIXME not sure if owner_uid is a good idea here, we may have a base entry without user entry (?) $result = db_query("SELECT plugin_data,title,content,link,tag_cache,author FROM ttrss_entries, ttrss_user_entries @@ -669,7 +646,7 @@ "author" => $entry_author, "stored" => $stored_article); - foreach ($pluginhost->get_hooks($pluginhost::HOOK_ARTICLE_FILTER) as $plugin) { + foreach ($pluginhost->get_hooks(PluginHost::HOOK_ARTICLE_FILTER) as $plugin) { $article = $plugin->hook_article_filter($article); } @@ -682,9 +659,7 @@ $entry_content = $article["content"]; // escaped below - if ($debug_enabled) { - _debug("update_rss_feed: plugin data: $entry_plugin_data"); - } + _debug("plugin data: $entry_plugin_data", $debug_enabled); if ($cache_images && is_writable(CACHE_DIR . '/images')) cache_images($entry_content, $site_url, $debug_enabled); @@ -700,9 +675,7 @@ if (db_num_rows($result) == 0) { - if ($debug_enabled) { - _debug("update_rss_feed: base guid [$entry_guid] not found"); - } + _debug("base guid [$entry_guid] not found", $debug_enabled); // base post entry does not exist, create it @@ -770,9 +743,7 @@ if (db_num_rows($result) == 1) { - if ($debug_enabled) { - _debug("update_rss_feed: base guid found, checking for user record"); - } + _debug("base guid found, checking for user record", $debug_enabled); // this will be used below in update handler $orig_content_hash = db_fetch_result($result, 0, "content_hash"); @@ -787,7 +758,7 @@ /* $stored_guid = db_fetch_result($result, 0, "guid"); if ($stored_guid != $entry_guid_hashed) { - if ($debug_enabled) _debug("upgrading compat guid to hashed one"); + if ($debug_enabled) _debug("upgrading compat guid to hashed one", $debug_enabled); db_query("UPDATE ttrss_entries SET guid = '$entry_guid_hashed' WHERE id = '$ref_id'"); @@ -809,7 +780,7 @@ $entry_tags); if ($debug_enabled) { - _debug("update_rss_feed: article filters: "); + _debug("article filters: ", $debug_enabled); if (count($article_filters) != 0) { print_r($article_filters); } @@ -822,9 +793,7 @@ $score = calculate_article_score($article_filters); - if ($debug_enabled) { - _debug("update_rss_feed: initial score: $score"); - } + _debug("initial score: $score", $debug_enabled); $query = "SELECT ref_id, int_id FROM ttrss_user_entries WHERE ref_id = '$ref_id' AND owner_uid = '$owner_uid' @@ -837,9 +806,7 @@ // okay it doesn't exist - create user entry if (db_num_rows($result) == 0) { - if ($debug_enabled) { - _debug("update_rss_feed: user record not found, creating..."); - } + _debug("user record not found, creating...", $debug_enabled); if ($score >= -500 && !find_article_filter($article_filters, 'catchup')) { $unread = 'true'; @@ -873,9 +840,7 @@ $ngram_similar = db_fetch_result($result, 0, "similar"); - if ($debug_enabled) { - _debug("update_rss_feed: N-gram similar results: $ngram_similar"); - } + _debug("N-gram similar results: $ngram_similar", $debug_enabled); if ($ngram_similar > 0) { $unread = 'false'; @@ -913,17 +878,13 @@ $entry_int_id = db_fetch_result($result, 0, "int_id"); } } else { - if ($debug_enabled) { - _debug("update_rss_feed: user record FOUND"); - } + _debug("user record FOUND", $debug_enabled); $entry_ref_id = db_fetch_result($result, 0, "ref_id"); $entry_int_id = db_fetch_result($result, 0, "int_id"); } - if ($debug_enabled) { - _debug("update_rss_feed: RID: $entry_ref_id, IID: $entry_int_id"); - } + _debug("RID: $entry_ref_id, IID: $entry_int_id", $debug_enabled); $post_needs_update = false; $update_insignificant = false; @@ -953,7 +914,7 @@ if ($post_needs_update) { if (defined('DAEMON_EXTENDED_DEBUG')) { - _debug("update_rss_feed: post $entry_guid_hashed needs update..."); + _debug("post $entry_guid_hashed needs update...", $debug_enabled); } // print ""; @@ -977,16 +938,12 @@ db_query("COMMIT"); - if ($debug_enabled) { - _debug("update_rss_feed: assigning labels..."); - } + _debug("assigning labels...", $debug_enabled); assign_article_to_label_filters($entry_ref_id, $article_filters, $owner_uid, $article_labels); - if ($debug_enabled) { - _debug("update_rss_feed: looking for enclosures..."); - } + _debug("looking for enclosures...", $debug_enabled); // enclosures @@ -1003,7 +960,7 @@ } if ($debug_enabled) { - _debug("update_rss_feed: article enclosures:"); + _debug("article enclosures:", $debug_enabled); print_r($enclosures); } @@ -1060,7 +1017,7 @@ $filtered_tags = array_unique($filtered_tags); if ($debug_enabled) { - _debug("update_rss_feed: filtered article tags:"); + _debug("filtered article tags:", $debug_enabled); print_r($filtered_tags); } @@ -1105,9 +1062,7 @@ } if (get_pref("AUTO_ASSIGN_LABELS", $owner_uid, false)) { - if ($debug_enabled) { - _debug("update_rss_feed: auto-assigning labels..."); - } + _debug("auto-assigning labels...", $debug_enabled); foreach ($labels as $label) { $caption = preg_quote($label["caption"]); @@ -1120,21 +1075,15 @@ } } - if ($debug_enabled) { - _debug("update_rss_feed: article processed"); - } + _debug("article processed", $debug_enabled); } if (!$last_updated) { - if ($debug_enabled) { - _debug("update_rss_feed: new feed, catching it up..."); - } + _debug("new feed, catching it up...", $debug_enabled); catchup_feed($feed, false, $owner_uid); } - if ($debug_enabled) { - _debug("purging feed..."); - } + _debug("purging feed...", $debug_enabled); purge_feed($feed, 0, $debug_enabled); @@ -1147,9 +1096,7 @@ $error_msg = db_escape_string(mb_substr($rss->error(), 0, 245)); - if ($debug_enabled) { - _debug("update_rss_feed: error fetching feed: $error_msg"); - } + _debug("error fetching feed: $error_msg", $debug_enabled); db_query( "UPDATE ttrss_feeds SET last_error = '$error_msg', @@ -1158,10 +1105,7 @@ unset($rss); - if ($debug_enabled) { - _debug("update_rss_feed: done"); - } - + _debug("done", $debug_enabled); } function cache_images($html, $site_url, $debug) { @@ -1221,7 +1165,7 @@ } function expire_lock_files($debug) { - if ($debug) _debug("Removing old lock files..."); + //if ($debug) _debug("Removing old lock files..."); $num_deleted = 0; @@ -1230,7 +1174,7 @@ if ($files) { foreach ($files as $file) { - if (!file_is_locked($file) && time() - filemtime($file) > 86400*2) { + if (!file_is_locked(basename($file)) && time() - filemtime($file) > 86400*2) { unlink($file); ++$num_deleted; } @@ -1238,14 +1182,14 @@ } } - if ($debug) _debug("Removed $num_deleted files."); + if ($debug) _debug("Removed $num_deleted old lock files."); } function expire_cached_files($debug) { foreach (array("simplepie", "images", "export", "upload") as $dir) { $cache_dir = CACHE_DIR . "/$dir"; - if ($debug) _debug("Expiring $cache_dir"); +// if ($debug) _debug("Expiring $cache_dir"); $num_deleted = 0; @@ -1263,7 +1207,7 @@ } } - if ($debug) _debug("Removed $num_deleted files."); + if ($debug) _debug("$cache_dir: removed $num_deleted files."); } } @@ -1417,5 +1361,26 @@ mb_strtolower(strip_tags($title), 'utf-8')); } + function verify_feed_xml($feed_data) { + libxml_use_internal_errors(true); + $doc = new DOMDocument(); + $doc->loadXML($feed_data); + $error = libxml_get_last_error(); + libxml_clear_errors(); + return $error; + } + function housekeeping_common($debug) { + expire_cached_files($debug); + expire_lock_files($debug); + expire_error_log($debug); + + $count = update_feedbrowser_cache(); + _debug("Feedbrowser updated, $count feeds processed."); + + purge_orphans( true); + $rc = cleanup_tags( 14, 50000); + + _debug("Cleaned $rc cached tags."); + } ?>