X-Git-Url: https://git.wh0rd.org/?a=blobdiff_plain;f=include%2Frssfuncs.php;h=754b4f4cc97fad7808241da9cd428c006b525eef;hb=af244f927e3464c595fd53021f315dd0c4c31ec0;hp=2be791cb7cb345d5a46fc4e7c51098f3cf4c3034;hpb=1e253d1ec9a93ca26e5e54c79a9e61d8e858102c;p=tt-rss.git diff --git a/include/rssfuncs.php b/include/rssfuncs.php index 2be791cb..754b4f4c 100644 --- a/include/rssfuncs.php +++ b/include/rssfuncs.php @@ -1,7 +1,20 @@ $v) { + if ($k != "feed" && isset($v)) { + $tmp .= sha1("$k:" . (is_array($v) ? implode(",", $v) : $v)); + } + } + + return sha1(implode(",", PluginHost::getInstance()->get_plugin_names())) . ":" . sha1($tmp); + } function update_feedbrowser_cache() { @@ -79,7 +92,7 @@ $login_thresh_qpart = ""; } - // Test if the feed need a update (update interval exceded). + // Test if the feed need a update (update interval exceeded). if (DB_TYPE == "pgsql") { $update_limit_qpart = "AND (( ttrss_feeds.update_interval = 0 @@ -88,8 +101,10 @@ ) OR ( ttrss_feeds.update_interval > 0 AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_feeds.update_interval || ' minutes') AS INTERVAL) - ) OR ttrss_feeds.last_updated IS NULL - OR last_updated = '1970-01-01 00:00:00')"; + ) OR (ttrss_feeds.last_updated IS NULL + AND ttrss_user_prefs.value != '-1') + OR (last_updated = '1970-01-01 00:00:00' + AND ttrss_user_prefs.value != '-1'))"; } else { $update_limit_qpart = "AND (( ttrss_feeds.update_interval = 0 @@ -98,34 +113,37 @@ ) OR ( ttrss_feeds.update_interval > 0 AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ttrss_feeds.update_interval MINUTE) - ) OR ttrss_feeds.last_updated IS NULL - OR last_updated = '1970-01-01 00:00:00')"; + ) OR (ttrss_feeds.last_updated IS NULL + AND ttrss_user_prefs.value != '-1') + OR (last_updated = '1970-01-01 00:00:00' + AND ttrss_user_prefs.value != '-1'))"; } // Test if feed is currently being updated by another process. if (DB_TYPE == "pgsql") { - $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '5 minutes')"; + $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '10 minutes')"; } else { - $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 5 MINUTE))"; + $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 10 MINUTE))"; } // Test if there is a limit to number of updated feeds $query_limit = ""; if($limit) $query_limit = sprintf("LIMIT %d", $limit); - $random_qpart = sql_random_function(); - - // We search for feed needing update. - $result = db_query("SELECT DISTINCT ttrss_feeds.feed_url,$random_qpart + $query = "SELECT DISTINCT ttrss_feeds.feed_url, ttrss_feeds.last_updated FROM ttrss_feeds, ttrss_users, ttrss_user_prefs WHERE ttrss_feeds.owner_uid = ttrss_users.id + AND ttrss_user_prefs.profile IS NULL AND ttrss_users.id = ttrss_user_prefs.owner_uid AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' $login_thresh_qpart $update_limit_qpart $updstart_thresh_qpart - ORDER BY $random_qpart $query_limit"); + ORDER BY last_updated $query_limit"; + + // We search for feed needing update. + $result = db_query($query); if($debug) _debug(sprintf("Scheduled %d feeds to update...", db_num_rows($result))); @@ -150,10 +168,6 @@ WHERE feed_url IN (%s)", implode(',', $feeds_quoted))); } - expire_cached_files($debug); - expire_lock_files($debug); - expire_error_log($debug); - $nf = 0; // For each feed, we call the feed update function. @@ -164,11 +178,12 @@ // since we have the data cached, we can deal with other feeds with the same url - $tmp_result = db_query("SELECT DISTINCT ttrss_feeds.id,last_updated + $tmp_result = db_query("SELECT DISTINCT ttrss_feeds.id,last_updated,ttrss_feeds.owner_uid FROM ttrss_feeds, ttrss_users, ttrss_user_prefs WHERE ttrss_user_prefs.owner_uid = ttrss_feeds.owner_uid AND ttrss_users.id = ttrss_user_prefs.owner_uid AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' AND + ttrss_user_prefs.profile IS NULL AND feed_url = '".db_escape_string($feed)."' AND (ttrss_feeds.update_interval > 0 OR ttrss_user_prefs.value != '-1') @@ -176,9 +191,13 @@ ORDER BY ttrss_feeds.id $query_limit"); if (db_num_rows($tmp_result) > 0) { + $rss = false; + while ($tline = db_fetch_assoc($tmp_result)) { - if($debug) _debug(" => " . $tline["last_updated"] . ", " . $tline["id"]); - update_rss_feed($tline["id"], true); + if($debug) _debug(" => " . $tline["last_updated"] . ", " . $tline["id"] . " " . $tline["owner_uid"]); + + $rss = update_rss_feed($tline["id"], true, false); + _debug_suppress(false); ++$nf; } } @@ -194,19 +213,19 @@ } // function update_daemon_common // ignore_daemon is not used - function update_rss_feed($feed, $ignore_daemon = false, $no_cache = false, - $override_url = false) { - - require_once "lib/simplepie/simplepie.inc"; + function update_rss_feed($feed, $ignore_daemon = false, $no_cache = false, $rss = false) { $debug_enabled = defined('DAEMON_EXTENDED_DEBUG') || $_REQUEST['xdebug']; + _debug_suppress(!$debug_enabled); _debug("start", $debug_enabled); $result = db_query("SELECT id,update_interval,auth_login, - feed_url,auth_pass,cache_images,last_updated, + feed_url,auth_pass,cache_images, mark_unread_on_update, owner_uid, - pubsub_state, auth_pass_encrypted + pubsub_state, auth_pass_encrypted, + (SELECT max(date_entered) FROM + ttrss_entries, ttrss_user_entries where ref_id = id AND feed_id = '$feed') AS last_article_timestamp FROM ttrss_feeds WHERE id = '$feed'"); if (db_num_rows($result) == 0) { @@ -214,7 +233,11 @@ return false; } - $last_updated = db_fetch_result($result, 0, "last_updated"); + $last_article_timestamp = @strtotime(db_fetch_result($result, 0, "last_article_timestamp")); + + if (defined('_DISABLE_HTTP_304')) + $last_article_timestamp = 0; + $owner_uid = db_fetch_result($result, 0, "owner_uid"); $mark_unread_on_update = sql_bool_to_bool(db_fetch_result($result, 0, "mark_unread_on_update")); @@ -238,57 +261,57 @@ $feed = db_escape_string($feed); - if ($override_url) $fetch_url = $override_url; - $date_feed_processed = date('Y-m-d H:i'); - $cache_filename = CACHE_DIR . "/simplepie/" . sha1($fetch_url) . ".feed"; + $cache_filename = CACHE_DIR . "/simplepie/" . sha1($fetch_url) . ".xml"; - // Ignore cache if new feed or manual update. - $cache_age = ($no_cache || is_null($last_updated) || $last_updated == '1970-01-01 00:00:00') ? - 30 : get_feed_update_interval($feed) * 60; + $pluginhost = new PluginHost(); + $pluginhost->set_debug($debug_enabled); + $user_plugins = get_pref("_ENABLED_PLUGINS", $owner_uid); - _debug("cache filename: $cache_filename exists: " . file_exists($cache_filename), $debug_enabled); - _debug("cache age: $cache_age; no cache: $no_cache", $debug_enabled); + $pluginhost->load(PLUGINS, PluginHost::KIND_ALL); + $pluginhost->load($user_plugins, PluginHost::KIND_USER, $owner_uid); + $pluginhost->load_data(); - $cached_feed_data_hash = false; + if ($rss && is_object($rss) && get_class($rss) == "FeedParser") { + _debug("using previously initialized parser object"); + } else { + $rss_hash = false; - $rss = false; - $rss_hash = false; - $cache_timestamp = file_exists($cache_filename) ? filemtime($cache_filename) : 0; - $last_updated_timestamp = strtotime($last_updated); + $force_refetch = isset($_REQUEST["force_refetch"]); - $force_refetch = isset($_REQUEST["force_refetch"]); + foreach ($pluginhost->get_hooks(PluginHost::HOOK_FETCH_FEED) as $plugin) { + $feed_data = $plugin->hook_fetch_feed($feed_data, $fetch_url, $owner_uid, $feed, $last_article_timestamp, $auth_login, $auth_pass); + } - if (file_exists($cache_filename) && - is_readable($cache_filename) && - !$auth_login && !$auth_pass && - filemtime($cache_filename) > time() - $cache_age) { + // try cache + if (!$feed_data && + file_exists($cache_filename) && + is_readable($cache_filename) && + !$auth_login && !$auth_pass && + filemtime($cache_filename) > time() - 30) { - _debug("using local cache.", $debug_enabled); + _debug("using local cache [$cache_filename].", $debug_enabled); - if ($cache_timestamp > $last_updated_timestamp) { - @$rss_data = file_get_contents($cache_filename); + @$feed_data = file_get_contents($cache_filename); - if ($rss_data) { - $rss_hash = sha1($rss_data); - @$rss = unserialize($rss_data); - } - } else if (!$force_refetch) { - _debug("local cache valid and older than last_updated, nothing to do.", $debug_enabled); - return; + if ($feed_data) { + $rss_hash = sha1($feed_data); } - } - if (!$rss) { + } else { + _debug("local cache will not be used for this feed", $debug_enabled); + } + // fetch feed from source if (!$feed_data) { - _debug("fetching [$fetch_url] (ts: $cache_timestamp/$last_updated_timestamp)", $debug_enabled); + _debug("fetching [$fetch_url]...", $debug_enabled); + _debug("If-Modified-Since: ".gmdate('D, d M Y H:i:s \G\M\T', $last_article_timestamp), $debug_enabled); $feed_data = fetch_file_contents($fetch_url, false, $auth_login, $auth_pass, false, $no_cache ? FEED_FETCH_NO_CACHE_TIMEOUT : FEED_FETCH_TIMEOUT, - $force_refetch ? 0 : max($last_updated_timestamp, $cache_timestamp)); + $force_refetch ? 0 : $last_article_timestamp); global $fetch_curl_used; @@ -302,7 +325,7 @@ _debug("fetch done.", $debug_enabled); - if ($feed_data) { + /* if ($feed_data) { $error = verify_feed_xml($feed_data); if ($error) { @@ -318,6 +341,16 @@ if ($error) $feed_data = ''; } } + } */ + + // cache vanilla feed data for re-use + if ($feed_data && !$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) { + $new_rss_hash = sha1($feed_data); + + if ($new_rss_hash != $rss_hash) { + _debug("saving $cache_filename", $debug_enabled); + @file_put_contents($cache_filename, $feed_data); + } } } @@ -344,29 +377,24 @@ } } - $pluginhost = new PluginHost(); - $pluginhost->set_debug($debug_enabled, $debug_enabled); - $user_plugins = get_pref("_ENABLED_PLUGINS", $owner_uid); - - $pluginhost->load(PLUGINS, PluginHost::KIND_ALL); - $pluginhost->load($user_plugins, PluginHost::KIND_USER, $owner_uid); - $pluginhost->load_data(); - foreach ($pluginhost->get_hooks(PluginHost::HOOK_FEED_FETCHED) as $plugin) { - $feed_data = $plugin->hook_feed_fetched($feed_data); + $feed_data = $plugin->hook_feed_fetched($feed_data, $fetch_url, $owner_uid, $feed); } + // set last update to now so if anything *simplepie* crashes later we won't be + // continuously failing on the same feed + //db_query("UPDATE ttrss_feeds SET last_updated = NOW() WHERE id = '$feed'"); + if (!$rss) { - $rss = new SimplePie(); - $rss->set_sanitize_class("SanitizeDummy"); - // simplepie ignores the above and creates default sanitizer anyway, - // so let's override it... - $rss->sanitize = new SanitizeDummy(); - $rss->set_output_encoding('UTF-8'); - $rss->set_raw_data($feed_data); - $rss->enable_cache(false); - - @$rss->init(); + $rss = new FeedParser($feed_data); + $rss->init(); + } + + if (DETECT_ARTICLE_LANGUAGE) { + require_once "lib/languagedetect/LanguageDetect.php"; + + $lang = new Text_LanguageDetect(); + $lang->setNameMode(2); } // print_r($rss); @@ -375,17 +403,6 @@ if (!$rss->error()) { - // cache data for later - if (!$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) { - $rss_data = serialize($rss); - $new_rss_hash = sha1($rss_data); - - if ($new_rss_hash != $rss_hash) { - _debug("saving $cache_filename", $debug_enabled); - @file_put_contents($cache_filename, serialize($rss)); - } - } - // We use local pluginhost here because we need to load different per-user feed plugins $pluginhost->run_hooks(PluginHost::HOOK_FEED_PARSED, "hook_feed_parsed", $rss); @@ -399,7 +416,7 @@ $favicon_interval_qpart = "favicon_last_checked < DATE_SUB(NOW(), INTERVAL 12 HOUR)"; } - $result = db_query("SELECT title,site_url,owner_uid, + $result = db_query("SELECT title,site_url,owner_uid,favicon_avg_color, (favicon_last_checked IS NULL OR $favicon_interval_qpart) AS favicon_needs_check FROM ttrss_feeds WHERE id = '$feed'"); @@ -408,24 +425,43 @@ $orig_site_url = db_fetch_result($result, 0, "site_url"); $favicon_needs_check = sql_bool_to_bool(db_fetch_result($result, 0, "favicon_needs_check")); + $favicon_avg_color = db_fetch_result($result, 0, "favicon_avg_color"); $owner_uid = db_fetch_result($result, 0, "owner_uid"); $site_url = db_escape_string(mb_substr(rewrite_relative_url($fetch_url, $rss->get_link()), 0, 245)); + _debug("site_url: $site_url", $debug_enabled); + _debug("feed_title: " . $rss->get_title(), $debug_enabled); + if ($favicon_needs_check || $force_refetch) { - _debug("checking favicon...", $debug_enabled); - check_feed_favicon($site_url, $feed, $link); + /* terrible hack: if we crash on floicon shit here, we won't check + * the icon avgcolor again (unless the icon got updated) */ + $favicon_file = ICONS_DIR . "/$feed.ico"; + $favicon_modified = @filemtime($favicon_file); + + _debug("checking favicon...", $debug_enabled); - if (file_exists($favicon_file) && function_exists("imagecreatefromstring")) { + check_feed_favicon($site_url, $feed); + $favicon_modified_new = @filemtime($favicon_file); + + if ($favicon_modified_new > $favicon_modified) + $favicon_avg_color = ''; + + if (file_exists($favicon_file) && function_exists("imagecreatefromstring") && $favicon_avg_color == '') { require_once "colors.php"; + db_query("UPDATE ttrss_feeds SET favicon_avg_color = 'fail' WHERE + id = '$feed'"); + $favicon_color = db_escape_string( calculate_avg_color($favicon_file)); $favicon_colorstring = ",favicon_avg_color = '".$favicon_color."'"; + } else if ($favicon_avg_color == 'fail') { + _debug("floicon failed on this file, not trying to recalculate avg color", $debug_enabled); } db_query("UPDATE ttrss_feeds SET favicon_last_checked = NOW() @@ -437,10 +473,12 @@ $feed_title = db_escape_string($rss->get_title()); - _debug("registering title: $feed_title", $debug_enabled); + if ($feed_title) { + _debug("registering title: $feed_title", $debug_enabled); - db_query("UPDATE ttrss_feeds SET - title = '$feed_title' WHERE id = '$feed'"); + db_query("UPDATE ttrss_feeds SET + title = '$feed_title' WHERE id = '$feed'"); + } } if ($site_url && $orig_site_url != $site_url) { @@ -483,7 +521,20 @@ _debug("feed hub url: $feed_hub_url", $debug_enabled); - if ($feed_hub_url && function_exists('curl_init') && + $feed_self_url = $fetch_url; + + $links = $rss->get_links('self'); + + if ($links && is_array($links)) { + foreach ($links as $l) { + $feed_self_url = $l; + break; + } + } + + _debug("feed self url = $feed_self_url"); + + if ($feed_hub_url && $feed_self_url && function_exists('curl_init') && !ini_get("open_basedir")) { require_once 'lib/pubsubhubbub/subscriber.php'; @@ -493,9 +544,9 @@ $s = new Subscriber($feed_hub_url, $callback_url); - $rc = $s->subscribe($fetch_url); + $rc = $s->subscribe($feed_self_url); - _debug("feed hub url found, subscribe request sent.", $debug_enabled); + _debug("feed hub url found, subscribe request sent. [rc=$rc]", $debug_enabled); db_query("UPDATE ttrss_feeds SET pubsub_state = 1 WHERE id = '$feed'"); @@ -513,6 +564,8 @@ if (!$entry_guid) $entry_guid = $item->get_link(); if (!$entry_guid) $entry_guid = make_guid_from_title($item->get_title()); + _debug("f_guid $entry_guid", $debug_enabled); + if (!$entry_guid) continue; $entry_guid = "$owner_uid,$entry_guid"; @@ -523,20 +576,21 @@ $entry_timestamp = ""; - $entry_timestamp = strtotime($item->get_date()); + $entry_timestamp = $item->get_date(); + + _debug("orig date: " . $item->get_date(), $debug_enabled); if ($entry_timestamp == -1 || !$entry_timestamp || $entry_timestamp > time()) { $entry_timestamp = time(); - $no_orig_date = 'true'; - } else { - $no_orig_date = 'false'; } $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); _debug("date $entry_timestamp [$entry_timestamp_fmt]", $debug_enabled); - $entry_title = html_entity_decode($item->get_title(), ENT_COMPAT, 'UTF-8'); +// $entry_title = html_entity_decode($item->get_title(), ENT_COMPAT, 'UTF-8'); +// $entry_title = decode_numeric_entities($entry_title); + $entry_title = $item->get_title(); $entry_link = rewrite_relative_url($site_url, $item->get_link()); @@ -554,30 +608,39 @@ print "\n"; } - $entry_comments = $item->data["comments"]; + $entry_language = ""; + + if (DETECT_ARTICLE_LANGUAGE) { + $entry_language = $lang->detect($entry_title . " " . $entry_content, 1); + + if (count($entry_language) > 0) { + $entry_language = array_keys($entry_language); + + // the fuck? + if (is_array($entry_language)) + $entry_language = ""; + else + $entry_language = db_escape_string(substr($entry_language[0], 0, 2)); - if ($item->get_author()) { - $entry_author_item = $item->get_author(); - $entry_author = $entry_author_item->get_name(); - if (!$entry_author) $entry_author = $entry_author_item->get_email(); + _debug("detected language: $entry_language", $debug_enabled); + } else { + $entry_language = ""; + } } + $entry_comments = $item->get_comments_url(); + $entry_author = $item->get_author(); + $entry_guid = db_escape_string(mb_substr($entry_guid, 0, 245)); $entry_comments = db_escape_string(mb_substr(trim($entry_comments), 0, 245)); $entry_author = db_escape_string(mb_substr(trim($entry_author), 0, 245)); - $num_comments = $item->get_item_tags('http://purl.org/rss/1.0/modules/slash/', 'comments'); - - if (is_array($num_comments) && is_array($num_comments[0])) { - $num_comments = (int) $num_comments[0]["data"]; - } else { - $num_comments = 0; - } + $num_comments = (int) $item->get_comments_count(); _debug("author $entry_author", $debug_enabled); _debug("num_comments: $num_comments", $debug_enabled); - _debug("looking for tags [1]...", $debug_enabled); + _debug("looking for tags...", $debug_enabled); // parse entries into tags @@ -587,38 +650,28 @@ if (is_array($additional_tags_src)) { foreach ($additional_tags_src as $tobj) { - array_push($additional_tags, $tobj->get_term()); + array_push($additional_tags, $tobj); } } - _debug("category tags:", $debug_enabled); - _debug("looking for tags [2]...", $debug_enabled); - $entry_tags = array_unique($additional_tags); for ($i = 0; $i < count($entry_tags); $i++) $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8'); - _debug("done collecting data.", $debug_enabled); - - // TODO: less memory-hungry implementation + _debug("tags found: " . join(",", $entry_tags), $debug_enabled); - _debug("applying plugin filters..", $debug_enabled); + _debug("done collecting data.", $debug_enabled); - // FIXME not sure if owner_uid is a good idea here, we may have a base entry without user entry (?) - $result = db_query("SELECT plugin_data,title,content,link,tag_cache,author FROM ttrss_entries, ttrss_user_entries - WHERE ref_id = id AND (guid = '".db_escape_string($entry_guid)."' OR guid = '$entry_guid_hashed') AND owner_uid = $owner_uid"); + $result = db_query("SELECT id, content_hash FROM ttrss_entries + WHERE guid = '".db_escape_string($entry_guid)."' OR guid = '$entry_guid_hashed'"); if (db_num_rows($result) != 0) { - $entry_plugin_data = db_fetch_result($result, 0, "plugin_data"); - $stored_article = array("title" => db_fetch_result($result, 0, "title"), - "content" => db_fetch_result($result, 0, "content"), - "link" => db_fetch_result($result, 0, "link"), - "tags" => explode(",", db_fetch_result($result, 0, "tag_cache")), - "author" => db_fetch_result($result, 0, "author")); + $base_entry_id = db_fetch_result($result, 0, "id"); + $entry_stored_hash = db_fetch_result($result, 0, "content_hash"); } else { - $entry_plugin_data = ""; - $stored_article = array(); + $base_entry_id = false; + $entry_stored_hash = ""; } $article = array("owner_uid" => $owner_uid, // read only @@ -627,32 +680,62 @@ "content" => $entry_content, "link" => $entry_link, "tags" => $entry_tags, - "plugin_data" => $entry_plugin_data, "author" => $entry_author, - "stored" => $stored_article); + "feed" => array("id" => $feed, + "fetch_url" => $fetch_url, + "site_url" => $site_url) + ); + + $entry_plugin_data = ""; + $entry_current_hash = calculate_article_hash($article); + + _debug("article hash: $entry_current_hash [stored=$entry_stored_hash]", $debug_enabled); + + if ($entry_current_hash == $entry_stored_hash) { + _debug("stored article seems up to date [IID: $base_entry_id], updating timestamp only", $debug_enabled); + + // we keep encountering the entry in feeds, so we need to + // update date_updated column so that we don't get horrible + // dupes when the entry gets purged and reinserted again e.g. + // in the case of SLOW SLOW OMG SLOW updating feeds + + $base_entry_id = db_fetch_result($result, 0, "id"); + + db_query("UPDATE ttrss_entries SET date_updated = NOW() + WHERE id = '$base_entry_id'"); + + continue; + } + + _debug("hash differs, applying plugin filters:", $debug_enabled); foreach ($pluginhost->get_hooks(PluginHost::HOOK_ARTICLE_FILTER) as $plugin) { + _debug("... " . get_class($plugin), $debug_enabled); + + $start = microtime(true); $article = $plugin->hook_article_filter($article); + + _debug("=== " . sprintf("%.4f (sec)", microtime(true) - $start), $debug_enabled); + + $entry_plugin_data .= mb_strtolower(get_class($plugin)) . ","; } + $entry_plugin_data = db_escape_string($entry_plugin_data); + + _debug("plugin data: $entry_plugin_data", $debug_enabled); + $entry_tags = $article["tags"]; $entry_guid = db_escape_string($entry_guid); $entry_title = db_escape_string($article["title"]); $entry_author = db_escape_string($article["author"]); $entry_link = db_escape_string($article["link"]); - $entry_plugin_data = db_escape_string($article["plugin_data"]); $entry_content = $article["content"]; // escaped below - - _debug("plugin data: $entry_plugin_data", $debug_enabled); - if ($cache_images && is_writable(CACHE_DIR . '/images')) cache_images($entry_content, $site_url, $debug_enabled); $entry_content = db_escape_string($entry_content, false); - $content_hash = "SHA1:" . sha1($entry_content); - db_query("BEGIN"); $result = db_query("SELECT id FROM ttrss_entries @@ -672,13 +755,13 @@ updated, content, content_hash, - cached_content, no_orig_date, date_updated, date_entered, comments, num_comments, plugin_data, + lang, author) VALUES ('$entry_title', @@ -686,41 +769,27 @@ '$entry_link', '$entry_timestamp_fmt', '$entry_content', - '$content_hash', - '', - $no_orig_date, + '$entry_current_hash', + false, NOW(), '$date_feed_processed', '$entry_comments', '$num_comments', '$entry_plugin_data', + '$entry_language', '$entry_author')"); $article_labels = array(); } else { - // we keep encountering the entry in feeds, so we need to - // update date_updated column so that we don't get horrible - // dupes when the entry gets purged and reinserted again e.g. - // in the case of SLOW SLOW OMG SLOW updating feeds - $base_entry_id = db_fetch_result($result, 0, "id"); - db_query("UPDATE ttrss_entries SET date_updated = NOW() - WHERE id = '$base_entry_id'"); - $article_labels = get_article_labels($base_entry_id, $owner_uid); } // now it should exist, if not - bad luck then - $result = db_query("SELECT - id,content_hash,no_orig_date,title,plugin_data,guid, - ".SUBSTRING_FOR_DATE."(date_updated,1,19) as date_updated, - ".SUBSTRING_FOR_DATE."(updated,1,19) as updated, - num_comments - FROM - ttrss_entries + $result = db_query("SELECT id FROM ttrss_entries WHERE guid = '$entry_guid' OR guid = '$entry_guid_hashed'"); $entry_ref_id = 0; @@ -730,14 +799,6 @@ _debug("base guid found, checking for user record", $debug_enabled); - // this will be used below in update handler - $orig_content_hash = db_fetch_result($result, 0, "content_hash"); - $orig_title = db_fetch_result($result, 0, "title"); - $orig_num_comments = db_fetch_result($result, 0, "num_comments"); - $orig_date_updated = strtotime(db_fetch_result($result, - 0, "date_updated")); - $orig_plugin_data = db_fetch_result($result, 0, "plugin_data"); - $ref_id = db_fetch_result($result, 0, "id"); $entry_ref_id = $ref_id; @@ -851,7 +912,7 @@ $p = new Publisher(PUBSUBHUBBUB_HUB); - $pubsub_result = $p->publish_update($rss_link); + /* $pubsub_result = */ $p->publish_update($rss_link); } $result = db_query( @@ -871,53 +932,20 @@ _debug("RID: $entry_ref_id, IID: $entry_int_id", $debug_enabled); - $post_needs_update = false; - $update_insignificant = false; - - if ($orig_num_comments != $num_comments) { - $post_needs_update = true; - $update_insignificant = true; - } - - if ($entry_plugin_data != $orig_plugin_data) { - $post_needs_update = true; - $update_insignificant = true; - } - - if ($content_hash != $orig_content_hash) { - $post_needs_update = true; - $update_insignificant = false; - } - - if (db_escape_string($orig_title) != $entry_title) { - $post_needs_update = true; - $update_insignificant = false; - } - - // if post needs update, update it and mark all user entries - // linking to this post as updated - if ($post_needs_update) { - - if (defined('DAEMON_EXTENDED_DEBUG')) { - _debug("post $entry_guid_hashed needs update...", $debug_enabled); - } - -// print ""; - - db_query("UPDATE ttrss_entries - SET title = '$entry_title', content = '$entry_content', - content_hash = '$content_hash', - updated = '$entry_timestamp_fmt', - num_comments = '$num_comments', - plugin_data = '$entry_plugin_data' - WHERE id = '$ref_id'"); - - if (!$update_insignificant) { - if ($mark_unread_on_update) { - db_query("UPDATE ttrss_user_entries - SET last_read = null, unread = true WHERE ref_id = '$ref_id'"); - } - } + db_query("UPDATE ttrss_entries + SET title = '$entry_title', + content = '$entry_content', + content_hash = '$entry_current_hash', + updated = '$entry_timestamp_fmt', + num_comments = '$num_comments', + plugin_data = '$entry_plugin_data', + author = '$entry_author', + lang = '$entry_language' + WHERE id = '$ref_id'"); + + if ($mark_unread_on_update) { + db_query("UPDATE ttrss_user_entries + SET last_read = null, unread = true WHERE ref_id = '$ref_id'"); } } @@ -939,7 +967,7 @@ if (is_array($encs)) { foreach ($encs as $e) { $e_item = array( - $e->link, $e->type, $e->length); + $e->link, $e->type, $e->length, $e->title, $e->width, $e->height); array_push($enclosures, $e_item); } } @@ -951,18 +979,24 @@ db_query("BEGIN"); +// debugging +// db_query("DELETE FROM ttrss_enclosures WHERE post_id = '$entry_ref_id'"); + foreach ($enclosures as $enc) { $enc_url = db_escape_string($enc[0]); $enc_type = db_escape_string($enc[1]); $enc_dur = db_escape_string($enc[2]); + $enc_title = db_escape_string($enc[3]); + $enc_width = intval($enc[4]); + $enc_height = intval($enc[5]); $result = db_query("SELECT id FROM ttrss_enclosures WHERE content_url = '$enc_url' AND post_id = '$entry_ref_id'"); if (db_num_rows($result) == 0) { db_query("INSERT INTO ttrss_enclosures - (content_url, content_type, title, duration, post_id) VALUES - ('$enc_url', '$enc_type', '', '$enc_dur', '$entry_ref_id')"); + (content_url, content_type, title, duration, post_id, width, height) VALUES + ('$enc_url', '$enc_type', '$enc_title', '$enc_dur', '$entry_ref_id', $enc_width, $enc_height)"); } } @@ -1063,11 +1097,6 @@ _debug("article processed", $debug_enabled); } - if (!$last_updated) { - _debug("new feed, catching it up...", $debug_enabled); - catchup_feed($feed, false, $owner_uid); - } - _debug("purging feed...", $debug_enabled); purge_feed($feed, 0, $debug_enabled); @@ -1081,21 +1110,27 @@ $error_msg = db_escape_string(mb_substr($rss->error(), 0, 245)); - _debug("error fetching feed: $error_msg", $debug_enabled); + _debug("fetch error: $error_msg", $debug_enabled); + + if (count($rss->errors()) > 1) { + foreach ($rss->errors() as $error) { + _debug("+ $error"); + } + } db_query( "UPDATE ttrss_feeds SET last_error = '$error_msg', - last_updated = NOW() WHERE id = '$feed'"); - } + last_updated = NOW() WHERE id = '$feed'"); - unset($rss); + unset($rss); + } _debug("done", $debug_enabled); + + return $rss; } function cache_images($html, $site_url, $debug) { - $cache_dir = CACHE_DIR . "/images"; - libxml_use_internal_errors(true); $charset_hack = ' @@ -1119,21 +1154,20 @@ if (!file_exists($local_filename)) { $file_content = fetch_file_contents($src); - if ($file_content && strlen($file_content) > 1024) { + if ($file_content && strlen($file_content) > _MIN_CACHE_IMAGE_SIZE) { file_put_contents($local_filename, $file_content); } } - if (file_exists($local_filename)) { + /* if (file_exists($local_filename)) { $entry->setAttribute('src', SELF_URL_PATH . '/image.php?url=' . base64_encode($src)); - } + } */ } } - $node = $doc->getElementsByTagName('body')->item(0); - - return $doc->saveXML($node); + //$node = $doc->getElementsByTagName('body')->item(0); + //return $doc->saveXML($node); } function expire_error_log($debug) { @@ -1150,7 +1184,7 @@ } function expire_lock_files($debug) { - if ($debug) _debug("Removing old lock files..."); + //if ($debug) _debug("Removing old lock files..."); $num_deleted = 0; @@ -1167,14 +1201,14 @@ } } - if ($debug) _debug("Removed $num_deleted files."); + if ($debug) _debug("Removed $num_deleted old lock files."); } function expire_cached_files($debug) { foreach (array("simplepie", "images", "export", "upload") as $dir) { $cache_dir = CACHE_DIR . "/$dir"; - if ($debug) _debug("Expiring $cache_dir"); +// if ($debug) _debug("Expiring $cache_dir"); $num_deleted = 0; @@ -1192,7 +1226,7 @@ } } - if ($debug) _debug("Removed $num_deleted files."); + if ($debug) _debug("$cache_dir: removed $num_deleted files."); } } @@ -1226,7 +1260,7 @@ foreach ($filter["rules"] as $rule) { $match = false; - $reg_exp = $rule["reg_exp"]; + $reg_exp = str_replace('/', '\/', $rule["reg_exp"]); $rule_inverse = $rule["inverse"]; if (!$reg_exp) @@ -1255,8 +1289,12 @@ $match = @preg_match("/$reg_exp/i", $author); break; case "tag": - $tag_string = join(",", $tags); - $match = @preg_match("/$reg_exp/i", $tag_string); + foreach ($tags as $tag) { + if (@preg_match("/$reg_exp/i", $tag)) { + $match = true; + break; + } + } break; } @@ -1346,13 +1384,29 @@ mb_strtolower(strip_tags($title), 'utf-8')); } - function verify_feed_xml($feed_data) { + /* function verify_feed_xml($feed_data) { libxml_use_internal_errors(true); $doc = new DOMDocument(); $doc->loadXML($feed_data); $error = libxml_get_last_error(); libxml_clear_errors(); return $error; - } + } */ + + function housekeeping_common($debug) { + expire_cached_files($debug); + expire_lock_files($debug); + expire_error_log($debug); + $count = update_feedbrowser_cache(); + _debug("Feedbrowser updated, $count feeds processed."); + + purge_orphans( true); + $rc = cleanup_tags( 14, 50000); + + _debug("Cleaned $rc cached tags."); + + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_HOUSE_KEEPING, "hook_house_keeping", ""); + + } ?>