X-Git-Url: https://git.wh0rd.org/?a=blobdiff_plain;f=include%2Frssfuncs.php;h=1c1f18269b64a73e8638f61b9cfdc813ffc8cd14;hb=6383c9c53a280e50216fea88a9852fee46b41229;hp=43c02c7f4a1eb8baf0624e78cf365c74a3f02d9d;hpb=d8179cb9d927bb1bd3e1bed76a3c0cfaf43fd122;p=tt-rss.git diff --git a/include/rssfuncs.php b/include/rssfuncs.php index 43c02c7f..1c1f1826 100644 --- a/include/rssfuncs.php +++ b/include/rssfuncs.php @@ -2,6 +2,19 @@ define_default('DAEMON_UPDATE_LOGIN_LIMIT', 30); define_default('DAEMON_FEED_LIMIT', 500); define_default('DAEMON_SLEEP_INTERVAL', 120); + define_default('_MIN_CACHE_IMAGE_SIZE', 1024); + + function calculate_article_hash($article, $pluginhost) { + $tmp = ""; + + foreach ($article as $k => $v) { + if ($k != "feed" && isset($v)) { + $tmp .= sha1("$k:" . (is_array($v) ? implode(",", $v) : $v)); + } + } + + return sha1(implode(",", $pluginhost->get_plugin_names()) . $tmp); + } function update_feedbrowser_cache() { @@ -79,7 +92,7 @@ $login_thresh_qpart = ""; } - // Test if the feed need a update (update interval exceded). + // Test if the feed need a update (update interval exceeded). if (DB_TYPE == "pgsql") { $update_limit_qpart = "AND (( ttrss_feeds.update_interval = 0 @@ -88,8 +101,10 @@ ) OR ( ttrss_feeds.update_interval > 0 AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_feeds.update_interval || ' minutes') AS INTERVAL) - ) OR ttrss_feeds.last_updated IS NULL - OR last_updated = '1970-01-01 00:00:00')"; + ) OR (ttrss_feeds.last_updated IS NULL + AND ttrss_user_prefs.value != '-1') + OR (last_updated = '1970-01-01 00:00:00' + AND ttrss_user_prefs.value != '-1'))"; } else { $update_limit_qpart = "AND (( ttrss_feeds.update_interval = 0 @@ -98,8 +113,10 @@ ) OR ( ttrss_feeds.update_interval > 0 AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ttrss_feeds.update_interval MINUTE) - ) OR ttrss_feeds.last_updated IS NULL - OR last_updated = '1970-01-01 00:00:00')"; + ) OR (ttrss_feeds.last_updated IS NULL + AND ttrss_user_prefs.value != '-1') + OR (last_updated = '1970-01-01 00:00:00' + AND ttrss_user_prefs.value != '-1'))"; } // Test if feed is currently being updated by another process. @@ -118,6 +135,7 @@ ttrss_feeds, ttrss_users, ttrss_user_prefs WHERE ttrss_feeds.owner_uid = ttrss_users.id + AND ttrss_user_prefs.profile IS NULL AND ttrss_users.id = ttrss_user_prefs.owner_uid AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' $login_thresh_qpart $update_limit_qpart @@ -151,6 +169,7 @@ } $nf = 0; + $bstarted = microtime(true); // For each feed, we call the feed update function. foreach ($feeds_to_update as $feed) { @@ -165,6 +184,7 @@ ttrss_user_prefs.owner_uid = ttrss_feeds.owner_uid AND ttrss_users.id = ttrss_user_prefs.owner_uid AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' AND + ttrss_user_prefs.profile IS NULL AND feed_url = '".db_escape_string($feed)."' AND (ttrss_feeds.update_interval > 0 OR ttrss_user_prefs.value != '-1') @@ -176,13 +196,23 @@ while ($tline = db_fetch_assoc($tmp_result)) { if($debug) _debug(" => " . $tline["last_updated"] . ", " . $tline["id"] . " " . $tline["owner_uid"]); - $rss = update_rss_feed($tline["id"], true, false, $rss); + + $fstarted = microtime(true); + $rss = update_rss_feed($tline["id"], true, false); _debug_suppress(false); + + _debug(sprintf(" %.4f (sec)", microtime(true) - $fstarted)); + ++$nf; } } } + if ($nf > 0) { + _debug(sprintf("Processed %d feeds in %.4f (sec), %.4f (sec/feed avg)", $nf, + microtime(true) - $bstarted, (microtime(true) - $bstarted) / $nf)); + } + require_once "digest.php"; // Send feed digests by email if needed. @@ -201,7 +231,7 @@ _debug("start", $debug_enabled); $result = db_query("SELECT id,update_interval,auth_login, - feed_url,auth_pass,cache_images,last_updated, + feed_url,auth_pass,cache_images, mark_unread_on_update, owner_uid, pubsub_state, auth_pass_encrypted, (SELECT max(date_entered) FROM @@ -213,7 +243,6 @@ return false; } - $last_updated = db_fetch_result($result, 0, "last_updated"); $last_article_timestamp = @strtotime(db_fetch_result($result, 0, "last_article_timestamp")); if (defined('_DISABLE_HTTP_304')) @@ -261,12 +290,18 @@ $force_refetch = isset($_REQUEST["force_refetch"]); - if (file_exists($cache_filename) && + foreach ($pluginhost->get_hooks(PluginHost::HOOK_FETCH_FEED) as $plugin) { + $feed_data = $plugin->hook_fetch_feed($feed_data, $fetch_url, $owner_uid, $feed, $last_article_timestamp, $auth_login, $auth_pass); + } + + // try cache + if (!$feed_data && + file_exists($cache_filename) && is_readable($cache_filename) && !$auth_login && !$auth_pass && filemtime($cache_filename) > time() - 30) { - _debug("using local cache.", $debug_enabled); + _debug("using local cache [$cache_filename].", $debug_enabled); @$feed_data = file_get_contents($cache_filename); @@ -277,14 +312,8 @@ } else { _debug("local cache will not be used for this feed", $debug_enabled); } - } - - if (!$rss) { - - foreach ($pluginhost->get_hooks(PluginHost::HOOK_FETCH_FEED) as $plugin) { - $feed_data = $plugin->hook_fetch_feed($feed_data, $fetch_url, $owner_uid, $feed); - } + // fetch feed from source if (!$feed_data) { _debug("fetching [$fetch_url]...", $debug_enabled); _debug("If-Modified-Since: ".gmdate('D, d M Y H:i:s \G\M\T', $last_article_timestamp), $debug_enabled); @@ -323,6 +352,16 @@ } } } */ + + // cache vanilla feed data for re-use + if ($feed_data && !$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) { + $new_rss_hash = sha1($feed_data); + + if ($new_rss_hash != $rss_hash) { + _debug("saving $cache_filename", $debug_enabled); + @file_put_contents($cache_filename, $feed_data); + } + } } if (!$feed_data) { @@ -374,16 +413,6 @@ if (!$rss->error()) { - // cache data for later - if (!$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) { - $new_rss_hash = sha1($rss_data); - - if ($new_rss_hash != $rss_hash && count($rss->get_items()) > 0 ) { - _debug("saving $cache_filename", $debug_enabled); - @file_put_contents($cache_filename, $feed_data); - } - } - // We use local pluginhost here because we need to load different per-user feed plugins $pluginhost->run_hooks(PluginHost::HOOK_FEED_PARSED, "hook_feed_parsed", $rss); @@ -452,7 +481,7 @@ if (!$registered_title || $registered_title == "[Unknown]") { - $feed_title = db_escape_string($rss->get_title()); + $feed_title = db_escape_string(mb_substr($rss->get_title(), 0, 199); if ($feed_title) { _debug("registering title: $feed_title", $debug_enabled); @@ -527,7 +556,7 @@ $rc = $s->subscribe($feed_self_url); - _debug("feed hub url found, subscribe request sent.", $debug_enabled); + _debug("feed hub url found, subscribe request sent. [rc=$rc]", $debug_enabled); db_query("UPDATE ttrss_feeds SET pubsub_state = 1 WHERE id = '$feed'"); @@ -544,9 +573,6 @@ $entry_guid = $item->get_id(); if (!$entry_guid) $entry_guid = $item->get_link(); if (!$entry_guid) $entry_guid = make_guid_from_title($item->get_title()); - - _debug("f_guid $entry_guid", $debug_enabled); - if (!$entry_guid) continue; $entry_guid = "$owner_uid,$entry_guid"; @@ -563,9 +589,6 @@ if ($entry_timestamp == -1 || !$entry_timestamp || $entry_timestamp > time()) { $entry_timestamp = time(); - $no_orig_date = 'true'; - } else { - $no_orig_date = 'false'; } $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); @@ -598,10 +621,12 @@ $entry_language = $lang->detect($entry_title . " " . $entry_content, 1); if (count($entry_language) > 0) { - $entry_language = array_keys($entry_language); - $entry_language = db_escape_string(substr($entry_language[0], 0, 2)); + $possible = array_keys($entry_language); + $entry_language = $possible[0]; _debug("detected language: $entry_language", $debug_enabled); + } else { + $entry_language = ""; } } @@ -640,24 +665,15 @@ _debug("done collecting data.", $debug_enabled); - // TODO: less memory-hungry implementation - - _debug("applying plugin filters..", $debug_enabled); - - // FIXME not sure if owner_uid is a good idea here, we may have a base entry without user entry (?) - $result = db_query("SELECT plugin_data,title,content,link,tag_cache,author FROM ttrss_entries, ttrss_user_entries - WHERE ref_id = id AND (guid = '".db_escape_string($entry_guid)."' OR guid = '$entry_guid_hashed') AND owner_uid = $owner_uid"); + $result = db_query("SELECT id, content_hash FROM ttrss_entries + WHERE guid = '".db_escape_string($entry_guid)."' OR guid = '$entry_guid_hashed'"); if (db_num_rows($result) != 0) { - $entry_plugin_data = db_fetch_result($result, 0, "plugin_data"); - $stored_article = array("title" => db_fetch_result($result, 0, "title"), - "content" => db_fetch_result($result, 0, "content"), - "link" => db_fetch_result($result, 0, "link"), - "tags" => explode(",", db_fetch_result($result, 0, "tag_cache")), - "author" => db_fetch_result($result, 0, "author")); + $base_entry_id = db_fetch_result($result, 0, "id"); + $entry_stored_hash = db_fetch_result($result, 0, "content_hash"); } else { - $entry_plugin_data = ""; - $stored_article = array(); + $base_entry_id = false; + $entry_stored_hash = ""; } $article = array("owner_uid" => $owner_uid, // read only @@ -666,36 +682,63 @@ "content" => $entry_content, "link" => $entry_link, "tags" => $entry_tags, - "plugin_data" => $entry_plugin_data, "author" => $entry_author, - "stored" => $stored_article, + "language" => $entry_language, // read only "feed" => array("id" => $feed, "fetch_url" => $fetch_url, "site_url" => $site_url) ); + $entry_plugin_data = ""; + $entry_current_hash = calculate_article_hash($article, $pluginhost); + + _debug("article hash: $entry_current_hash [stored=$entry_stored_hash]", $debug_enabled); + + if ($entry_current_hash == $entry_stored_hash && !isset($_REQUEST["force_rehash"])) { + _debug("stored article seems up to date [IID: $base_entry_id], updating timestamp only", $debug_enabled); + + // we keep encountering the entry in feeds, so we need to + // update date_updated column so that we don't get horrible + // dupes when the entry gets purged and reinserted again e.g. + // in the case of SLOW SLOW OMG SLOW updating feeds + + $base_entry_id = db_fetch_result($result, 0, "id"); + + db_query("UPDATE ttrss_entries SET date_updated = NOW() + WHERE id = '$base_entry_id'"); + + continue; + } + + _debug("hash differs, applying plugin filters:", $debug_enabled); + foreach ($pluginhost->get_hooks(PluginHost::HOOK_ARTICLE_FILTER) as $plugin) { + _debug("... " . get_class($plugin), $debug_enabled); + + $start = microtime(true); $article = $plugin->hook_article_filter($article); + + _debug("=== " . sprintf("%.4f (sec)", microtime(true) - $start), $debug_enabled); + + $entry_plugin_data .= mb_strtolower(get_class($plugin)) . ","; } + $entry_plugin_data = db_escape_string($entry_plugin_data); + + _debug("plugin data: $entry_plugin_data", $debug_enabled); + $entry_tags = $article["tags"]; $entry_guid = db_escape_string($entry_guid); $entry_title = db_escape_string($article["title"]); $entry_author = db_escape_string($article["author"]); $entry_link = db_escape_string($article["link"]); - $entry_plugin_data = db_escape_string($article["plugin_data"]); $entry_content = $article["content"]; // escaped below - - _debug("plugin data: $entry_plugin_data", $debug_enabled); - if ($cache_images && is_writable(CACHE_DIR . '/images')) cache_images($entry_content, $site_url, $debug_enabled); $entry_content = db_escape_string($entry_content, false); - $content_hash = "SHA1:" . sha1($entry_content); - db_query("BEGIN"); $result = db_query("SELECT id FROM ttrss_entries @@ -729,8 +772,8 @@ '$entry_link', '$entry_timestamp_fmt', '$entry_content', - '$content_hash', - $no_orig_date, + '$entry_current_hash', + false, NOW(), '$date_feed_processed', '$entry_comments', @@ -742,28 +785,14 @@ $article_labels = array(); } else { - // we keep encountering the entry in feeds, so we need to - // update date_updated column so that we don't get horrible - // dupes when the entry gets purged and reinserted again e.g. - // in the case of SLOW SLOW OMG SLOW updating feeds - $base_entry_id = db_fetch_result($result, 0, "id"); - db_query("UPDATE ttrss_entries SET date_updated = NOW() - WHERE id = '$base_entry_id'"); - $article_labels = get_article_labels($base_entry_id, $owner_uid); } // now it should exist, if not - bad luck then - $result = db_query("SELECT - id,content_hash,no_orig_date,title,plugin_data,guid, - ".SUBSTRING_FOR_DATE."(date_updated,1,19) as date_updated, - ".SUBSTRING_FOR_DATE."(updated,1,19) as updated, - num_comments - FROM - ttrss_entries + $result = db_query("SELECT id FROM ttrss_entries WHERE guid = '$entry_guid' OR guid = '$entry_guid_hashed'"); $entry_ref_id = 0; @@ -773,14 +802,6 @@ _debug("base guid found, checking for user record", $debug_enabled); - // this will be used below in update handler - $orig_content_hash = db_fetch_result($result, 0, "content_hash"); - $orig_title = db_fetch_result($result, 0, "title"); - $orig_num_comments = db_fetch_result($result, 0, "num_comments"); - $orig_date_updated = strtotime(db_fetch_result($result, - 0, "date_updated")); - $orig_plugin_data = db_fetch_result($result, 0, "plugin_data"); - $ref_id = db_fetch_result($result, 0, "id"); $entry_ref_id = $ref_id; @@ -894,7 +915,7 @@ $p = new Publisher(PUBSUBHUBBUB_HUB); - $pubsub_result = $p->publish_update($rss_link); + /* $pubsub_result = */ $p->publish_update($rss_link); } $result = db_query( @@ -914,53 +935,20 @@ _debug("RID: $entry_ref_id, IID: $entry_int_id", $debug_enabled); - $post_needs_update = false; - $update_insignificant = false; - - if ($orig_num_comments != $num_comments) { - $post_needs_update = true; - $update_insignificant = true; - } - - if ($entry_plugin_data != $orig_plugin_data) { - $post_needs_update = true; - $update_insignificant = true; - } - - if ($content_hash != $orig_content_hash) { - $post_needs_update = true; - $update_insignificant = false; - } - - if (db_escape_string($orig_title) != $entry_title) { - $post_needs_update = true; - $update_insignificant = false; - } - - // if post needs update, update it and mark all user entries - // linking to this post as updated - if ($post_needs_update) { - - if (defined('DAEMON_EXTENDED_DEBUG')) { - _debug("post $entry_guid_hashed needs update...", $debug_enabled); - } - -// print ""; - - db_query("UPDATE ttrss_entries - SET title = '$entry_title', content = '$entry_content', - content_hash = '$content_hash', - updated = '$entry_timestamp_fmt', - num_comments = '$num_comments', - plugin_data = '$entry_plugin_data' - WHERE id = '$ref_id'"); - - if (!$update_insignificant) { - if ($mark_unread_on_update) { - db_query("UPDATE ttrss_user_entries - SET last_read = null, unread = true WHERE ref_id = '$ref_id'"); - } - } + db_query("UPDATE ttrss_entries + SET title = '$entry_title', + content = '$entry_content', + content_hash = '$entry_current_hash', + updated = '$entry_timestamp_fmt', + num_comments = '$num_comments', + plugin_data = '$entry_plugin_data', + author = '$entry_author', + lang = '$entry_language' + WHERE id = '$ref_id'"); + + if ($mark_unread_on_update) { + db_query("UPDATE ttrss_user_entries + SET last_read = null, unread = true WHERE ref_id = '$ref_id'"); } } @@ -982,7 +970,7 @@ if (is_array($encs)) { foreach ($encs as $e) { $e_item = array( - $e->link, $e->type, $e->length, $e->title); + $e->link, $e->type, $e->length, $e->title, $e->width, $e->height); array_push($enclosures, $e_item); } } @@ -1002,14 +990,16 @@ $enc_type = db_escape_string($enc[1]); $enc_dur = db_escape_string($enc[2]); $enc_title = db_escape_string($enc[3]); + $enc_width = intval($enc[4]); + $enc_height = intval($enc[5]); $result = db_query("SELECT id FROM ttrss_enclosures WHERE content_url = '$enc_url' AND post_id = '$entry_ref_id'"); if (db_num_rows($result) == 0) { db_query("INSERT INTO ttrss_enclosures - (content_url, content_type, title, duration, post_id) VALUES - ('$enc_url', '$enc_type', '$enc_title', '$enc_dur', '$entry_ref_id')"); + (content_url, content_type, title, duration, post_id, width, height) VALUES + ('$enc_url', '$enc_type', '$enc_title', '$enc_dur', '$entry_ref_id', $enc_width, $enc_height)"); } } @@ -1144,8 +1134,6 @@ } function cache_images($html, $site_url, $debug) { - $cache_dir = CACHE_DIR . "/images"; - libxml_use_internal_errors(true); $charset_hack = ' @@ -1169,7 +1157,7 @@ if (!file_exists($local_filename)) { $file_content = fetch_file_contents($src); - if ($file_content && strlen($file_content) > 1024) { + if ($file_content && strlen($file_content) > _MIN_CACHE_IMAGE_SIZE) { file_put_contents($local_filename, $file_content); } }