define_default('DAEMON_SLEEP_INTERVAL', 120);
define_default('_MIN_CACHE_IMAGE_SIZE', 1024);
+ function calculate_article_hash($article, $pluginhost) {
+ $tmp = "";
+
+ foreach ($article as $k => $v) {
+ if ($k != "feed" && isset($v)) {
+ $tmp .= sha1("$k:" . (is_array($v) ? implode(",", $v) : $v));
+ }
+ }
+
+ return sha1(implode(",", $pluginhost->get_plugin_names()) . $tmp);
+ }
+
function update_feedbrowser_cache() {
$result = db_query("SELECT feed_url, site_url, title, COUNT(id) AS subscribers
}
$nf = 0;
+ $bstarted = microtime(true);
// For each feed, we call the feed update function.
foreach ($feeds_to_update as $feed) {
while ($tline = db_fetch_assoc($tmp_result)) {
if($debug) _debug(" => " . $tline["last_updated"] . ", " . $tline["id"] . " " . $tline["owner_uid"]);
+ $fstarted = microtime(true);
$rss = update_rss_feed($tline["id"], true, false);
_debug_suppress(false);
+
+ _debug(sprintf(" %.4f (sec)", microtime(true) - $fstarted));
+
++$nf;
}
}
}
+ if ($nf > 0) {
+ _debug(sprintf("Processed %d feeds in %.4f (sec), %.4f (sec/feed avg)", $nf,
+ microtime(true) - $bstarted, (microtime(true) - $bstarted) / $nf));
+ }
+
require_once "digest.php";
// Send feed digests by email if needed.
if (!$registered_title || $registered_title == "[Unknown]") {
- $feed_title = db_escape_string($rss->get_title());
+ $feed_title = db_escape_string(mb_substr($rss->get_title(), 0, 199));
if ($feed_title) {
_debug("registering title: $feed_title", $debug_enabled);
$entry_guid = $item->get_id();
if (!$entry_guid) $entry_guid = $item->get_link();
if (!$entry_guid) $entry_guid = make_guid_from_title($item->get_title());
-
- _debug("f_guid $entry_guid", $debug_enabled);
-
if (!$entry_guid) continue;
$entry_guid = "$owner_uid,$entry_guid";
$entry_language = $lang->detect($entry_title . " " . $entry_content, 1);
if (count($entry_language) > 0) {
- $entry_language = array_keys($entry_language);
-
- // the fuck?
- if (is_array($entry_language))
- $entry_language = "";
- else
- $entry_language = db_escape_string(substr($entry_language[0], 0, 2));
+ $possible = array_keys($entry_language);
+ $entry_language = $possible[0];
_debug("detected language: $entry_language", $debug_enabled);
} else {
_debug("done collecting data.", $debug_enabled);
- // TODO: less memory-hungry implementation
-
- _debug("applying plugin filters..", $debug_enabled);
-
- // FIXME not sure if owner_uid is a good idea here, we may have a base entry without user entry (?)
- $result = db_query("SELECT plugin_data,title,content,link,tag_cache,author FROM ttrss_entries, ttrss_user_entries
- WHERE ref_id = id AND (guid = '".db_escape_string($entry_guid)."' OR guid = '$entry_guid_hashed') AND owner_uid = $owner_uid");
+ $result = db_query("SELECT id, content_hash FROM ttrss_entries
+ WHERE guid = '".db_escape_string($entry_guid)."' OR guid = '$entry_guid_hashed'");
if (db_num_rows($result) != 0) {
- $entry_plugin_data = db_fetch_result($result, 0, "plugin_data");
- $stored_article = array("title" => db_fetch_result($result, 0, "title"),
- "content" => db_fetch_result($result, 0, "content"),
- "link" => db_fetch_result($result, 0, "link"),
- "tags" => explode(",", db_fetch_result($result, 0, "tag_cache")),
- "author" => db_fetch_result($result, 0, "author"));
+ $base_entry_id = db_fetch_result($result, 0, "id");
+ $entry_stored_hash = db_fetch_result($result, 0, "content_hash");
+ $article_labels = get_article_labels($base_entry_id, $owner_uid);
} else {
- $entry_plugin_data = "";
- $stored_article = array();
+ $base_entry_id = false;
+ $entry_stored_hash = "";
+ $article_labels = array();
}
$article = array("owner_uid" => $owner_uid, // read only
"title" => $entry_title,
"content" => $entry_content,
"link" => $entry_link,
+ "labels" => $article_labels, // current limitation: can add labels to article, can't remove them
"tags" => $entry_tags,
- "plugin_data" => $entry_plugin_data,
"author" => $entry_author,
- "stored" => $stored_article,
+ "force_catchup" => false, // ugly hack for the time being
+ "language" => $entry_language, // read only
"feed" => array("id" => $feed,
"fetch_url" => $fetch_url,
"site_url" => $site_url)
);
+ $entry_plugin_data = "";
+ $entry_current_hash = calculate_article_hash($article, $pluginhost);
+
+ _debug("article hash: $entry_current_hash [stored=$entry_stored_hash]", $debug_enabled);
+
+ if ($entry_current_hash == $entry_stored_hash && !isset($_REQUEST["force_rehash"])) {
+ _debug("stored article seems up to date [IID: $base_entry_id], updating timestamp only", $debug_enabled);
+
+ // we keep encountering the entry in feeds, so we need to
+ // update date_updated column so that we don't get horrible
+ // dupes when the entry gets purged and reinserted again e.g.
+ // in the case of SLOW SLOW OMG SLOW updating feeds
+
+ $base_entry_id = db_fetch_result($result, 0, "id");
+
+ db_query("UPDATE ttrss_entries SET date_updated = NOW()
+ WHERE id = '$base_entry_id'");
+
+ // if we allow duplicate posts, we have to continue to
+ // create the user entries for this feed
+ if (!get_pref("ALLOW_DUPLICATE_POSTS", $owner_uid, false)) {
+ continue;
+ }
+ }
+
+ _debug("hash differs, applying plugin filters:", $debug_enabled);
+
foreach ($pluginhost->get_hooks(PluginHost::HOOK_ARTICLE_FILTER) as $plugin) {
+ _debug("... " . get_class($plugin), $debug_enabled);
+
+ $start = microtime(true);
$article = $plugin->hook_article_filter($article);
+
+ _debug("=== " . sprintf("%.4f (sec)", microtime(true) - $start), $debug_enabled);
+
+ $entry_plugin_data .= mb_strtolower(get_class($plugin)) . ",";
}
+ $entry_plugin_data = db_escape_string($entry_plugin_data);
+
+ _debug("plugin data: $entry_plugin_data", $debug_enabled);
+
$entry_tags = $article["tags"];
$entry_guid = db_escape_string($entry_guid);
$entry_title = db_escape_string($article["title"]);
$entry_author = db_escape_string($article["author"]);
$entry_link = db_escape_string($article["link"]);
- $entry_plugin_data = db_escape_string($article["plugin_data"]);
$entry_content = $article["content"]; // escaped below
+ $entry_force_catchup = $article["force_catchup"];
+ $article_labels = $article["labels"];
+ if ($debug_enabled) {
+ _debug("article labels:", $debug_enabled);
+ print_r($article_labels);
+ }
- _debug("plugin data: $entry_plugin_data", $debug_enabled);
+ _debug("force catchup: $entry_force_catchup");
if ($cache_images && is_writable(CACHE_DIR . '/images'))
cache_images($entry_content, $site_url, $debug_enabled);
$entry_content = db_escape_string($entry_content, false);
- $content_hash = "SHA1:" . sha1($entry_content);
-
db_query("BEGIN");
$result = db_query("SELECT id FROM ttrss_entries
'$entry_link',
'$entry_timestamp_fmt',
'$entry_content',
- '$content_hash',
+ '$entry_current_hash',
false,
NOW(),
'$date_feed_processed',
'$entry_language',
'$entry_author')");
- $article_labels = array();
-
} else {
- // we keep encountering the entry in feeds, so we need to
- // update date_updated column so that we don't get horrible
- // dupes when the entry gets purged and reinserted again e.g.
- // in the case of SLOW SLOW OMG SLOW updating feeds
-
$base_entry_id = db_fetch_result($result, 0, "id");
-
- db_query("UPDATE ttrss_entries SET date_updated = NOW()
- WHERE id = '$base_entry_id'");
-
- $article_labels = get_article_labels($base_entry_id, $owner_uid);
}
// now it should exist, if not - bad luck then
- $result = db_query("SELECT
- id,content_hash,title,plugin_data,guid,
- num_comments
- FROM
- ttrss_entries
+ $result = db_query("SELECT id FROM ttrss_entries
WHERE guid = '$entry_guid' OR guid = '$entry_guid_hashed'");
$entry_ref_id = 0;
_debug("base guid found, checking for user record", $debug_enabled);
- // this will be used below in update handler
- $orig_content_hash = db_fetch_result($result, 0, "content_hash");
- $orig_title = db_fetch_result($result, 0, "title");
- $orig_num_comments = db_fetch_result($result, 0, "num_comments");
- $orig_plugin_data = db_fetch_result($result, 0, "plugin_data");
-
$ref_id = db_fetch_result($result, 0, "id");
$entry_ref_id = $ref_id;
_debug("user record not found, creating...", $debug_enabled);
- if ($score >= -500 && !find_article_filter($article_filters, 'catchup')) {
+ if ($score >= -500 && !find_article_filter($article_filters, 'catchup') && !$entry_force_catchup) {
$unread = 'true';
$last_read_qpart = 'NULL';
} else {
// N-grams
- if (DB_TYPE == "pgsql" and defined('_NGRAM_TITLE_DUPLICATE_THRESHOLD')) {
+ /* if (DB_TYPE == "pgsql" and defined('_NGRAM_TITLE_DUPLICATE_THRESHOLD')) {
$result = db_query("SELECT COUNT(*) AS similar FROM
ttrss_entries,ttrss_user_entries
if ($ngram_similar > 0) {
$unread = 'false';
}
- }
+ } */
$last_marked = ($marked == 'true') ? 'NOW()' : 'NULL';
$last_published = ($published == 'true') ? 'NOW()' : 'NULL';
_debug("RID: $entry_ref_id, IID: $entry_int_id", $debug_enabled);
- $post_needs_update = false;
- $update_insignificant = false;
-
- if ($orig_num_comments != $num_comments) {
- $post_needs_update = true;
- $update_insignificant = true;
- }
-
- if ($entry_plugin_data != $orig_plugin_data) {
- $post_needs_update = true;
- $update_insignificant = true;
+ db_query("UPDATE ttrss_entries
+ SET title = '$entry_title',
+ content = '$entry_content',
+ content_hash = '$entry_current_hash',
+ updated = '$entry_timestamp_fmt',
+ num_comments = '$num_comments',
+ plugin_data = '$entry_plugin_data',
+ author = '$entry_author',
+ lang = '$entry_language'
+ WHERE id = '$ref_id'");
+
+ if ($mark_unread_on_update) {
+ db_query("UPDATE ttrss_user_entries
+ SET last_read = null, unread = true WHERE ref_id = '$ref_id'");
}
+ }
- if ($content_hash != $orig_content_hash) {
- $post_needs_update = true;
- $update_insignificant = false;
- }
-
- if (db_escape_string($orig_title) != $entry_title) {
- $post_needs_update = true;
- $update_insignificant = false;
- }
-
- // if post needs update, update it and mark all user entries
- // linking to this post as updated
- if ($post_needs_update) {
-
- if (defined('DAEMON_EXTENDED_DEBUG')) {
- _debug("post $entry_guid_hashed needs update...", $debug_enabled);
- }
-
-// print "<!-- post $orig_title needs update : $post_needs_update -->";
+ db_query("COMMIT");
- db_query("UPDATE ttrss_entries
- SET title = '$entry_title', content = '$entry_content',
- content_hash = '$content_hash',
- updated = '$entry_timestamp_fmt',
- num_comments = '$num_comments',
- plugin_data = '$entry_plugin_data'
- WHERE id = '$ref_id'");
+ _debug("assigning labels [other]...", $debug_enabled);
- if (!$update_insignificant) {
- if ($mark_unread_on_update) {
- db_query("UPDATE ttrss_user_entries
- SET last_read = null, unread = true WHERE ref_id = '$ref_id'");
- }
- }
- }
+ foreach ($article_labels as $label) {
+ label_add_article($entry_ref_id, $label[1], $owner_uid);
}
- db_query("COMMIT");
-
- _debug("assigning labels...", $debug_enabled);
+ _debug("assigning labels [filters]...", $debug_enabled);
assign_article_to_label_filters($entry_ref_id, $article_filters,
$owner_uid, $article_labels);
db_query("COMMIT");
}
- if (get_pref("AUTO_ASSIGN_LABELS", $owner_uid, false)) {
- _debug("auto-assigning labels...", $debug_enabled);
-
- foreach ($labels as $label) {
- $caption = preg_quote($label["caption"]);
-
- if ($caption && preg_match("/\b$caption\b/i", "$tags_str " . strip_tags($entry_content) . " $entry_title")) {
- if (!labels_contains_caption($article_labels, $caption)) {
- label_add_article($entry_ref_id, $caption, $owner_uid);
- }
- }
- }
- }
-
_debug("article processed", $debug_enabled);
}
file_put_contents($local_filename, $file_content);
}
}
-
- /* if (file_exists($local_filename)) {
- $entry->setAttribute('src', SELF_URL_PATH . '/image.php?url=' .
- base64_encode($src));
- } */
}
}
-
- //$node = $doc->getElementsByTagName('body')->item(0);
- //return $doc->saveXML($node);
}
function expire_error_log($debug) {
return $error;
} */
+ function cleanup_counters_cache($debug) {
+ $result = db_query("DELETE FROM ttrss_counters_cache
+ WHERE feed_id > 0 AND
+ (SELECT COUNT(id) FROM ttrss_feeds WHERE
+ id = feed_id AND
+ ttrss_counters_cache.owner_uid = ttrss_feeds.owner_uid) = 0");
+ $frows = db_affected_rows($result);
+
+ $result = db_query("DELETE FROM ttrss_cat_counters_cache
+ WHERE feed_id > 0 AND
+ (SELECT COUNT(id) FROM ttrss_feed_categories WHERE
+ id = feed_id AND
+ ttrss_cat_counters_cache.owner_uid = ttrss_feed_categories.owner_uid) = 0");
+ $crows = db_affected_rows($result);
+
+ _debug("Removed $frows (feeds) $crows (cats) orphaned counter cache entries.");
+ }
+
function housekeeping_common($debug) {
expire_cached_files($debug);
expire_lock_files($debug);
_debug("Feedbrowser updated, $count feeds processed.");
purge_orphans( true);
+ cleanup_counters_cache($debug);
$rc = cleanup_tags( 14, 50000);
_debug("Cleaned $rc cached tags.");
git.wh0rd.org / tt-rss.git / blobdiff