cache downloaded feed data to minimize network access

diff --git a/include/rssfuncs.php b/include/rssfuncs.php
index d76ad7d7aec88f00df026841ffb49e02f14bf006..1fb2312dd95ebb1cd4ef62254130c1e998116840 100644 (file)
--- a/include/rssfuncs.php
+++ b/include/rssfuncs.php
@@ -119,7 +119,7 @@
                                AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL'
                                $login_thresh_qpart $update_limit_qpart
                         $updstart_thresh_qpart
-                       ORDER BY $random_qpart $query_limit");
+                       ORDER BY feed_url,$random_qpart $query_limit");
 
                $user_prefs_cache = array();
 
@@ -149,7 +149,6 @@
                        if($debug) _debug("Feed: " . $line["feed_url"] . ", " . $line["last_updated"]);
 
                        update_rss_feed($link, $line["id"], true);
-
                        sleep(1); // prevent flood (FIXME make this an option?)
                }
 
@@ -202,36 +201,44 @@
 
                $feed = db_escape_string($link, $feed);
 
-               /* if ($auth_login && $auth_pass ){
-                       $url_parts = array();
-                       preg_match("/(^[^:]*):\/\/(.*)/", $fetch_url, $url_parts);
+               if ($override_url) $fetch_url = $override_url;
 
-                       if ($url_parts[1] && $url_parts[2]) {
-                               $fetch_url = $url_parts[1] . "://$auth_login:$auth_pass@" . $url_parts[2];
-                       }
-               } */
+               $date_feed_processed = date('Y-m-d H:i');
 
-               if ($override_url)
-                       $fetch_url = $override_url;
+               $cache_filename = CACHE_DIR . "/simplepie/" . sha1($fetch_url) . ".xml";
+
+               // Ignore cache if new feed or manual update.
+               $cache_age = ($no_cache || is_null($last_updated) || $last_updated == '1970-01-01 00:00:00') ?
+                       30 : get_feed_update_interval($link, $feed) * 60;
 
                if ($debug_enabled) {
-                       _debug("update_rss_feed: fetching [$fetch_url]...");
+                       _debug("update_rss_feed: cache filename: $cache_filename exists: " . file_exists($cache_filename));
+                       _debug("update_rss_feed: cache age: $cache_age; no cache: $no_cache");
                }
 
-               // Ignore cache if new feed or manual update.
-               $cache_age = (is_null($last_updated) || $last_updated == '1970-01-01 00:00:00') ?
-                       -1 : get_feed_update_interval($link, $feed) * 60;
+               $cached_feed_data_hash = false;
 
-               $simplepie_cache_dir = CACHE_DIR . "/simplepie";
+               if (file_exists($cache_filename) &&
+                       is_readable($cache_filename) &&
+                       !$auth_login && !$auth_pass &&
+                       filemtime($cache_filename) > time() - $cache_age) {
 
-               $date_feed_processed = date('Y-m-d H:i');
+                               if ($debug_enabled) {
+                                       _debug("update_rss_feed: using local cache.");
+                               }
 
-               if (!is_dir($simplepie_cache_dir)) {
-                       mkdir($simplepie_cache_dir);
+                               $feed_data = file_get_contents($cache_filename);
+                               $cached_feed_data_hash = sha1($feed_data);
                }
 
-               $feed_data = fetch_file_contents($fetch_url, false,
-                       $auth_login, $auth_pass, false, $no_cache ? 15 : 45);
+               if (!$feed_data) {
+                       if ($debug_enabled) {
+                               _debug("update_rss_feed: fetching [$fetch_url]...");
+                       }
+
+                       $feed_data = fetch_file_contents($fetch_url, false,
+                               $auth_login, $auth_pass, false, $no_cache ? 15 : 45);
+               }
 
                if (!$feed_data) {
                        global $fetch_last_error;
@@ -249,6 +256,18 @@
                        return;
                }
 
+               // cache data for later
+               if (!$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) {
+                       if (sha1($feed_data) != $cached_feed_data_hash) {
+
+                               if ($debug_enabled) {
+                                       _debug("update_rss_feed: saving $cache_filename");
+                               }
+
+                               @file_put_contents($cache_filename, $feed_data);
+                       }
+               }
+
                $pluginhost = new PluginHost($link);
                $pluginhost->set_debug($debug_enabled);
                $user_plugins = get_pref($link, "_ENABLED_PLUGINS", $owner_uid);
@@ -272,19 +291,13 @@
                $rss->sanitize = new SanitizeDummy();
                $rss->set_output_encoding('UTF-8');
                $rss->set_raw_data($feed_data);
+               $rss->enable_cache(false);
 
                if ($debug_enabled) {
                        _debug("feed update interval (sec): " .
                                get_feed_update_interval($link, $feed)*60);
                }
 
-               $rss->enable_cache(!$no_cache);
-
-               if (!$no_cache) {
-                       $rss->set_cache_location($simplepie_cache_dir);
-                       $rss->set_cache_duration($cache_age);
-               }
-
                @$rss->init();
 
 //             print_r($rss);