X-Git-Url: https://git.wh0rd.org/?a=blobdiff_plain;f=include%2Frssfuncs.php;h=9ea2662fccd0ea07cef1b0e3f6a3344469dc0f02;hb=857efe49e653787f5408fc014ae71efec68494d5;hp=49d9e6009c448a8ab7d7e33281602b37df9a701c;hpb=17e74b21cf7dcb6d57c44b3cbf5e4c330f5f2c17;p=tt-rss.git diff --git a/include/rssfuncs.php b/include/rssfuncs.php index 49d9e600..9ea2662f 100644 --- a/include/rssfuncs.php +++ b/include/rssfuncs.php @@ -60,6 +60,12 @@ function update_daemon_common($link, $limit = DAEMON_FEED_LIMIT, $from_http = false, $debug = true) { // Process all other feeds using last_updated and interval parameters + $schema_version = get_schema_version($link); + + if ($schema_version != SCHEMA_VERSION) { + die("Schema version is wrong, please upgrade the database.\n"); + } + define('PREFS_NO_CACHE', true); // Test if the user has loggued in recently. If not, it does not update its feeds. @@ -77,6 +83,7 @@ if (DB_TYPE == "pgsql") { $update_limit_qpart = "AND (( ttrss_feeds.update_interval = 0 + AND ttrss_user_prefs.value != '-1' AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_user_prefs.value || ' minutes') AS INTERVAL) ) OR ( ttrss_feeds.update_interval > 0 @@ -86,6 +93,7 @@ } else { $update_limit_qpart = "AND (( ttrss_feeds.update_interval = 0 + AND ttrss_user_prefs.value != '-1' AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL CONVERT(ttrss_user_prefs.value, SIGNED INTEGER) MINUTE) ) OR ( ttrss_feeds.update_interval > 0 @@ -108,7 +116,7 @@ $random_qpart = sql_random_function(); // We search for feed needing update. - $result = db_query($link, "SELECT DISTINCT ttrss_feeds.feed_url + $result = db_query($link, "SELECT DISTINCT ttrss_feeds.feed_url,$random_qpart FROM ttrss_feeds, ttrss_users, ttrss_user_prefs WHERE @@ -117,7 +125,7 @@ AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' $login_thresh_qpart $update_limit_qpart $updstart_thresh_qpart - ORDER BY feed_url $query_limit"); + ORDER BY $random_qpart $query_limit"); $user_prefs_cache = array(); @@ -146,6 +154,9 @@ expire_cached_files($debug); expire_lock_files($debug); + expire_error_log($link, $debug); + + $nf = 0; // For each feed, we call the feed update function. foreach ($feeds_to_update as $feed) { @@ -155,18 +166,22 @@ // since we have the data cached, we can deal with other feeds with the same url - $tmp_result = db_query($link, "SELECT ttrss_feeds.feed_url,ttrss_feeds.id,last_updated - FROM ttrss_feeds, ttrss_users WHERE - ttrss_users.id = ttrss_feeds.owner_uid AND + $tmp_result = db_query($link, "SELECT DISTINCT ttrss_feeds.id,last_updated + FROM ttrss_feeds, ttrss_users, ttrss_user_prefs WHERE + ttrss_user_prefs.owner_uid = ttrss_feeds.owner_uid AND + ttrss_users.id = ttrss_user_prefs.owner_uid AND + ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' AND feed_url = '".db_escape_string($link, $feed)."' AND - ttrss_feeds.update_interval != -1 + (ttrss_feeds.update_interval > 0 OR + ttrss_user_prefs.value != '-1') $login_thresh_qpart - ORDER BY feed_url $query_limit"); + ORDER BY ttrss_feeds.id $query_limit"); if (db_num_rows($tmp_result) > 0) { while ($tline = db_fetch_assoc($tmp_result)) { if($debug) _debug(" => " . $tline["last_updated"] . ", " . $tline["id"]); update_rss_feed($link, $tline["id"], true); + ++$nf; } } } @@ -176,6 +191,8 @@ // Send feed digests by email if needed. send_headlines_digests($link, $debug); + return $nf; + } // function update_daemon_common // ignore_daemon is not used @@ -193,7 +210,7 @@ $result = db_query($link, "SELECT id,update_interval,auth_login, feed_url,auth_pass,cache_images,last_updated, mark_unread_on_update, owner_uid, - pubsub_state + pubsub_state, auth_pass_encrypted FROM ttrss_feeds WHERE id = '$feed'"); if (db_num_rows($result) == 0) { @@ -208,6 +225,8 @@ $mark_unread_on_update = sql_bool_to_bool(db_fetch_result($result, 0, "mark_unread_on_update")); $pubsub_state = db_fetch_result($result, 0, "pubsub_state"); + $auth_pass_encrypted = sql_bool_to_bool(db_fetch_result($result, + 0, "auth_pass_encrypted")); db_query($link, "UPDATE ttrss_feeds SET last_update_started = NOW() WHERE id = '$feed'"); @@ -215,6 +234,11 @@ $auth_login = db_fetch_result($result, 0, "auth_login"); $auth_pass = db_fetch_result($result, 0, "auth_pass"); + if ($auth_pass_encrypted) { + require_once "crypt.php"; + $auth_pass = decrypt_string($auth_pass); + } + $cache_images = sql_bool_to_bool(db_fetch_result($result, 0, "cache_images")); $fetch_url = db_fetch_result($result, 0, "feed_url"); @@ -242,6 +266,8 @@ $cache_timestamp = file_exists($cache_filename) ? filemtime($cache_filename) : 0; $last_updated_timestamp = strtotime($last_updated); + $force_refetch = isset($_REQUEST["force_refetch"]); + if (file_exists($cache_filename) && is_readable($cache_filename) && !$auth_login && !$auth_pass && @@ -258,7 +284,7 @@ $rss_hash = sha1($rss_data); @$rss = unserialize($rss_data); } - } else { + } else if (!$force_refetch) { if ($debug_enabled) { _debug("update_rss_feed: local cache valid and older than last_updated, nothing to do."); } @@ -274,8 +300,9 @@ } $feed_data = fetch_file_contents($fetch_url, false, - $auth_login, $auth_pass, false, $no_cache ? 15 : 45, - max($last_updated_timestamp, $cache_timestamp)); + $auth_login, $auth_pass, false, + $no_cache ? FEED_FETCH_NO_CACHE_TIMEOUT : FEED_FETCH_TIMEOUT, + $force_refetch ? 0 : max($last_updated_timestamp, $cache_timestamp)); if ($debug_enabled) { _debug("update_rss_feed: fetch done."); @@ -383,14 +410,25 @@ $site_url = db_escape_string($link, mb_substr(rewrite_relative_url($fetch_url, $rss->get_link()), 0, 245)); - if ($debug_enabled) { - _debug("update_rss_feed: checking favicon..."); - } + if ($favicon_needs_check || $force_refetch) { + if ($debug_enabled) { + _debug("update_rss_feed: checking favicon..."); + } - if ($favicon_needs_check) { check_feed_favicon($site_url, $feed, $link); + $favicon_file = ICONS_DIR . "/$feed.ico"; + + if (file_exists($favicon_file)) { + require_once "colors.php"; + + $favicon_color = db_escape_string($link, + calculate_avg_color($favicon_file)); + + $favicon_colorstring = ",favicon_avg_color = '".$favicon_color."'"; + } db_query($link, "UPDATE ttrss_feeds SET favicon_last_checked = NOW() + $favicon_colorstring WHERE id = '$feed'"); } @@ -486,14 +524,16 @@ if (!$entry_guid) $entry_guid = $item->get_link(); if (!$entry_guid) $entry_guid = make_guid_from_title($item->get_title()); - if ($debug_enabled) { - _debug("update_rss_feed: guid $entry_guid"); - } - if (!$entry_guid) continue; $entry_guid = "$owner_uid,$entry_guid"; + $entry_guid_hashed = db_escape_string($link, 'SHA1:' . sha1($entry_guid)); + + if ($debug_enabled) { + _debug("update_rss_feed: guid $entry_guid / $entry_guid_hashed"); + } + $entry_timestamp = ""; $entry_timestamp = strtotime($item->get_date()); @@ -511,7 +551,7 @@ _debug("update_rss_feed: date $entry_timestamp [$entry_timestamp_fmt]"); } - $entry_title = $item->get_title(); + $entry_title = html_entity_decode($item->get_title()); $entry_link = rewrite_relative_url($site_url, $item->get_link()); @@ -602,7 +642,7 @@ // FIXME not sure if owner_uid is a good idea here, we may have a base entry without user entry (?) $result = db_query($link, "SELECT plugin_data,title,content,link,tag_cache,author FROM ttrss_entries, ttrss_user_entries - WHERE ref_id = id AND guid = '".db_escape_string($link, $entry_guid)."' AND owner_uid = $owner_uid"); + WHERE ref_id = id AND (guid = '".db_escape_string($link, $entry_guid)."' OR guid = '$entry_guid_hashed') AND owner_uid = $owner_uid"); if (db_num_rows($result) != 0) { $entry_plugin_data = db_fetch_result($result, 0, "plugin_data"); @@ -653,7 +693,7 @@ db_query($link, "BEGIN"); $result = db_query($link, "SELECT id FROM ttrss_entries - WHERE guid = '$entry_guid'"); + WHERE (guid = '$entry_guid' OR guid = '$entry_guid_hashed')"); if (db_num_rows($result) == 0) { @@ -681,7 +721,7 @@ author) VALUES ('$entry_title', - '$entry_guid', + '$entry_guid_hashed', '$entry_link', '$entry_timestamp_fmt', '$entry_content', @@ -714,13 +754,13 @@ // now it should exist, if not - bad luck then $result = db_query($link, "SELECT - id,content_hash,no_orig_date,title,plugin_data, + id,content_hash,no_orig_date,title,plugin_data,guid, ".SUBSTRING_FOR_DATE."(date_updated,1,19) as date_updated, ".SUBSTRING_FOR_DATE."(updated,1,19) as updated, num_comments FROM ttrss_entries - WHERE guid = '$entry_guid'"); + WHERE guid = '$entry_guid' OR guid = '$entry_guid_hashed'"); $entry_ref_id = 0; $entry_int_id = 0; @@ -728,7 +768,7 @@ if (db_num_rows($result) == 1) { if ($debug_enabled) { - _debug("update_rss_feed: base guid [$entry_guid] found, checking for user record"); + _debug("update_rss_feed: base guid found, checking for user record"); } // this will be used below in update handler @@ -742,6 +782,14 @@ $ref_id = db_fetch_result($result, 0, "id"); $entry_ref_id = $ref_id; + /* $stored_guid = db_fetch_result($result, 0, "guid"); + if ($stored_guid != $entry_guid_hashed) { + if ($debug_enabled) _debug("upgrading compat guid to hashed one"); + + db_query($link, "UPDATE ttrss_entries SET guid = '$entry_guid_hashed' WHERE + id = '$ref_id'"); + } */ + // check for user post link to main table // do we allow duplicate posts with same GUID in different feeds? @@ -902,7 +950,7 @@ if ($post_needs_update) { if (defined('DAEMON_EXTENDED_DEBUG')) { - _debug("update_rss_feed: post $entry_guid needs update..."); + _debug("update_rss_feed: post $entry_guid_hashed needs update..."); } // print ""; @@ -1156,6 +1204,19 @@ return $doc->saveXML($node); } + function expire_error_log($link, $debug) { + if ($debug) _debug("Removing old error log entries..."); + + if (DB_TYPE == "pgsql") { + db_query($link, "DELETE FROM ttrss_error_log + WHERE created_at < NOW() - INTERVAL '7 days'"); + } else { + db_query($link, "DELETE FROM ttrss_error_log + WHERE created_at < DATE_SUB(NOW(), INTERVAL 7 DAY)"); + } + + } + function expire_lock_files($debug) { if ($debug) _debug("Removing old lock files..."); @@ -1178,7 +1239,7 @@ } function expire_cached_files($debug) { - foreach (array("simplepie", "images", "export") as $dir) { + foreach (array("simplepie", "images", "export", "upload") as $dir) { $cache_dir = CACHE_DIR . "/$dir"; if ($debug) _debug("Expiring $cache_dir"); @@ -1287,6 +1348,9 @@ if ($filter_match) { foreach ($filter["actions"] AS $action) { array_push($matches, $action); + + // if Stop action encountered, perform no further processing + if ($action["type"] == "stop") return $matches; } } }