| | 1 | <?php |
| | 2 | define('DAEMON_UPDATE_LOGIN_LIMIT', 30); |
| | 3 | define('DAEMON_FEED_LIMIT', 100); |
| | 4 | define('DAEMON_SLEEP_INTERVAL', 60); |
| | 5 | |
| | 6 | function update_feedbrowser_cache($link) { |
| | 7 | |
| | 8 | $result = db_query($link, "SELECT feed_url, site_url, title, COUNT(id) AS subscribers |
| | 9 | FROM ttrss_feeds WHERE (SELECT COUNT(id) = 0 FROM ttrss_feeds AS tf |
| | 10 | WHERE tf.feed_url = ttrss_feeds.feed_url |
| | 11 | AND (private IS true OR auth_login != '' OR auth_pass != '' OR feed_url LIKE '%:%@%/%')) |
| | 12 | GROUP BY feed_url, site_url, title ORDER BY subscribers DESC LIMIT 1000"); |
| | 13 | |
| | 14 | db_query($link, "BEGIN"); |
| | 15 | |
| | 16 | db_query($link, "DELETE FROM ttrss_feedbrowser_cache"); |
| | 17 | |
| | 18 | $count = 0; |
| | 19 | |
| | 20 | while ($line = db_fetch_assoc($result)) { |
| | 21 | $subscribers = db_escape_string($link, $line["subscribers"]); |
| | 22 | $feed_url = db_escape_string($link, $line["feed_url"]); |
| | 23 | $title = db_escape_string($link, $line["title"]); |
| | 24 | $site_url = db_escape_string($link, $line["site_url"]); |
| | 25 | |
| | 26 | $tmp_result = db_query($link, "SELECT subscribers FROM |
| | 27 | ttrss_feedbrowser_cache WHERE feed_url = '$feed_url'"); |
| | 28 | |
| | 29 | if (db_num_rows($tmp_result) == 0) { |
| | 30 | |
| | 31 | db_query($link, "INSERT INTO ttrss_feedbrowser_cache |
| | 32 | (feed_url, site_url, title, subscribers) VALUES ('$feed_url', |
| | 33 | '$site_url', '$title', '$subscribers')"); |
| | 34 | |
| | 35 | ++$count; |
| | 36 | |
| | 37 | } |
| | 38 | |
| | 39 | } |
| | 40 | |
| | 41 | db_query($link, "COMMIT"); |
| | 42 | |
| | 43 | return $count; |
| | 44 | |
| | 45 | } |
| | 46 | |
| | 47 | |
| | 48 | /** |
| | 49 | * Update a feed batch. |
| | 50 | * Used by daemons to update n feeds by run. |
| | 51 | * Only update feed needing a update, and not being processed |
| | 52 | * by another process. |
| | 53 | * |
| | 54 | * @param mixed $link Database link |
| | 55 | * @param integer $limit Maximum number of feeds in update batch. Default to DAEMON_FEED_LIMIT. |
| | 56 | * @param boolean $from_http Set to true if you call this function from http to disable cli specific code. |
| | 57 | * @param boolean $debug Set to false to disable debug output. Default to true. |
| | 58 | * @return void |
| | 59 | */ |
| | 60 | function update_daemon_common($link, $limit = DAEMON_FEED_LIMIT, $from_http = false, $debug = true) { |
| | 61 | // Process all other feeds using last_updated and interval parameters |
| | 62 | |
| | 63 | define('PREFS_NO_CACHE', true); |
| | 64 | |
| | 65 | // Test if the user has loggued in recently. If not, it does not update its feeds. |
| | 66 | if (!SINGLE_USER_MODE && DAEMON_UPDATE_LOGIN_LIMIT > 0) { |
| | 67 | if (DB_TYPE == "pgsql") { |
| | 68 | $login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'"; |
| | 69 | } else { |
| | 70 | $login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)"; |
| | 71 | } |
| | 72 | } else { |
| | 73 | $login_thresh_qpart = ""; |
| | 74 | } |
| | 75 | |
| | 76 | // Test if the feed need a update (update interval exceded). |
| | 77 | if (DB_TYPE == "pgsql") { |
| | 78 | $update_limit_qpart = "AND (( |
| | 79 | ttrss_feeds.update_interval = 0 |
| | 80 | AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_user_prefs.value || ' minutes') AS INTERVAL) |
| | 81 | ) OR ( |
| | 82 | ttrss_feeds.update_interval > 0 |
| | 83 | AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_feeds.update_interval || ' minutes') AS INTERVAL) |
| | 84 | ) OR ttrss_feeds.last_updated IS NULL |
| | 85 | OR last_updated = '1970-01-01 00:00:00')"; |
| | 86 | } else { |
| | 87 | $update_limit_qpart = "AND (( |
| | 88 | ttrss_feeds.update_interval = 0 |
| | 89 | AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL CONVERT(ttrss_user_prefs.value, SIGNED INTEGER) MINUTE) |
| | 90 | ) OR ( |
| | 91 | ttrss_feeds.update_interval > 0 |
| | 92 | AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ttrss_feeds.update_interval MINUTE) |
| | 93 | ) OR ttrss_feeds.last_updated IS NULL |
| | 94 | OR last_updated = '1970-01-01 00:00:00')"; |
| | 95 | } |
| | 96 | |
| | 97 | // Test if feed is currently being updated by another process. |
| | 98 | if (DB_TYPE == "pgsql") { |
| | 99 | $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '5 minutes')"; |
| | 100 | } else { |
| | 101 | $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 5 MINUTE))"; |
| | 102 | } |
| | 103 | |
| | 104 | // Test if there is a limit to number of updated feeds |
| | 105 | $query_limit = ""; |
| | 106 | if($limit) $query_limit = sprintf("LIMIT %d", $limit); |
| | 107 | |
| | 108 | $random_qpart = sql_random_function(); |
| | 109 | |
| | 110 | // We search for feed needing update. |
| | 111 | $result = db_query($link, "SELECT ttrss_feeds.feed_url,ttrss_feeds.id, ttrss_feeds.owner_uid, |
| | 112 | ".SUBSTRING_FOR_DATE."(ttrss_feeds.last_updated,1,19) AS last_updated, |
| | 113 | ttrss_feeds.update_interval |
| | 114 | FROM |
| | 115 | ttrss_feeds, ttrss_users, ttrss_user_prefs |
| | 116 | WHERE |
| | 117 | ttrss_feeds.owner_uid = ttrss_users.id |
| | 118 | AND ttrss_users.id = ttrss_user_prefs.owner_uid |
| | 119 | AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' |
| | 120 | $login_thresh_qpart $update_limit_qpart |
| | 121 | $updstart_thresh_qpart |
| | 122 | ORDER BY feed_url,$random_qpart $query_limit"); |
| | 123 | |
| | 124 | $user_prefs_cache = array(); |
| | 125 | |
| | 126 | if($debug) _debug(sprintf("Scheduled %d feeds to update...", db_num_rows($result))); |
| | 127 | |
| | 128 | // Here is a little cache magic in order to minimize risk of double feed updates. |
| | 129 | $feeds_to_update = array(); |
| | 130 | while ($line = db_fetch_assoc($result)) { |
| | 131 | $feeds_to_update[$line['id']] = $line; |
| | 132 | } |
| | 133 | |
| | 134 | // We update the feed last update started date before anything else. |
| | 135 | // There is no lag due to feed contents downloads |
| | 136 | // It prevent an other process to update the same feed. |
| | 137 | $feed_ids = array_keys($feeds_to_update); |
| | 138 | if($feed_ids) { |
| | 139 | db_query($link, sprintf("UPDATE ttrss_feeds SET last_update_started = NOW() |
| | 140 | WHERE id IN (%s)", implode(',', $feed_ids))); |
| | 141 | } |
| | 142 | |
| | 143 | expire_cached_files($debug); |
| | 144 | expire_lock_files($debug); |
| | 145 | |
| | 146 | // For each feed, we call the feed update function. |
| | 147 | while ($line = array_pop($feeds_to_update)) { |
| | 148 | |
| | 149 | if($debug) _debug("Feed: " . $line["feed_url"] . ", " . $line["last_updated"]); |
| | 150 | |
| | 151 | update_rss_feed($link, $line["id"], true); |
| | 152 | sleep(1); // prevent flood (FIXME make this an option?) |
| | 153 | } |
| | 154 | |
| | 155 | require_once "digest.php"; |
| | 156 | |
| | 157 | // Send feed digests by email if needed. |
| | 158 | send_headlines_digests($link, $debug); |
| | 159 | |
| | 160 | } // function update_daemon_common |
| | 161 | |
| | 162 | // ignore_daemon is not used |
| | 163 | function update_rss_feed($link, $feed, $ignore_daemon = false, $no_cache = false, |
| | 164 | $override_url = false) { |
| | 165 | |
| | 166 | require_once "lib/simplepie/simplepie.inc"; |
| | 167 | |
| | 168 | $debug_enabled = defined('DAEMON_EXTENDED_DEBUG') || $_REQUEST['xdebug']; |
| | 169 | |
| | 170 | if ($debug_enabled) { |
| | 171 | _debug("update_rss_feed: start"); |
| | 172 | } |
| | 173 | |
| | 174 | $result = db_query($link, "SELECT id,update_interval,auth_login, |
| | 175 | feed_url,auth_pass,cache_images,last_updated, |
| | 176 | mark_unread_on_update, owner_uid, |
| | 177 | pubsub_state |
| | 178 | FROM ttrss_feeds WHERE id = '$feed'"); |
| | 179 | |
| | 180 | if (db_num_rows($result) == 0) { |
| | 181 | if ($debug_enabled) { |
| | 182 | _debug("update_rss_feed: feed $feed NOT FOUND/SKIPPED"); |
| | 183 | } |
| | 184 | return false; |
| | 185 | } |
| | 186 | |
| | 187 | $last_updated = db_fetch_result($result, 0, "last_updated"); |
| | 188 | $owner_uid = db_fetch_result($result, 0, "owner_uid"); |
| | 189 | $mark_unread_on_update = sql_bool_to_bool(db_fetch_result($result, |
| | 190 | 0, "mark_unread_on_update")); |
| | 191 | $pubsub_state = db_fetch_result($result, 0, "pubsub_state"); |
| | 192 | |
| | 193 | db_query($link, "UPDATE ttrss_feeds SET last_update_started = NOW() |
| | 194 | WHERE id = '$feed'"); |
| | 195 | |
| | 196 | $auth_login = db_fetch_result($result, 0, "auth_login"); |
| | 197 | $auth_pass = db_fetch_result($result, 0, "auth_pass"); |
| | 198 | |
| | 199 | $cache_images = sql_bool_to_bool(db_fetch_result($result, 0, "cache_images")); |
| | 200 | $fetch_url = db_fetch_result($result, 0, "feed_url"); |
| | 201 | |
| | 202 | $feed = db_escape_string($link, $feed); |
| | 203 | |
| | 204 | if ($override_url) $fetch_url = $override_url; |
| | 205 | |
| | 206 | $date_feed_processed = date('Y-m-d H:i'); |
| | 207 | |
| | 208 | $cache_filename = CACHE_DIR . "/simplepie/" . sha1($fetch_url) . ".xml"; |
| | 209 | |
| | 210 | // Ignore cache if new feed or manual update. |
| | 211 | $cache_age = ($no_cache || is_null($last_updated) || $last_updated == '1970-01-01 00:00:00') ? |
| | 212 | 30 : get_feed_update_interval($link, $feed) * 60; |
| | 213 | |
| | 214 | if ($debug_enabled) { |
| | 215 | _debug("update_rss_feed: cache filename: $cache_filename exists: " . file_exists($cache_filename)); |
| | 216 | _debug("update_rss_feed: cache age: $cache_age; no cache: $no_cache"); |
| | 217 | } |
| | 218 | |
| | 219 | $cached_feed_data_hash = false; |
| | 220 | |
| | 221 | if (file_exists($cache_filename) && |
| | 222 | is_readable($cache_filename) && |
| | 223 | !$auth_login && !$auth_pass && |
| | 224 | filemtime($cache_filename) > time() - $cache_age) { |
| | 225 | |
| | 226 | if ($debug_enabled) { |
| | 227 | _debug("update_rss_feed: using local cache."); |
| | 228 | } |
| | 229 | |
| | 230 | $feed_data = file_get_contents($cache_filename); |
| | 231 | $cached_feed_data_hash = sha1($feed_data); |
| | 232 | } |
| | 233 | |
| | 234 | if (!$feed_data) { |
| | 235 | if ($debug_enabled) { |
| | 236 | _debug("update_rss_feed: fetching [$fetch_url]..."); |
| | 237 | } |
| | 238 | |
| | 239 | $feed_data = fetch_file_contents($fetch_url, false, |
| | 240 | $auth_login, $auth_pass, false, $no_cache ? 15 : 45); |
| | 241 | } |
| | 242 | |
| | 243 | if (!$feed_data) { |
| | 244 | global $fetch_last_error; |
| | 245 | |
| | 246 | if ($debug_enabled) { |
| | 247 | _debug("update_rss_feed: unable to fetch: $fetch_last_error"); |
| | 248 | } |
| | 249 | |
| | 250 | $error_escaped = db_escape_string($link, $fetch_last_error); |
| | 251 | |
| | 252 | db_query($link, |
| | 253 | "UPDATE ttrss_feeds SET last_error = '$error_escaped', |
| | 254 | last_updated = NOW() WHERE id = '$feed'"); |
| | 255 | |
| | 256 | return; |
| | 257 | } |
| | 258 | |
| | 259 | // cache data for later |
| | 260 | if (!$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) { |
| | 261 | if (sha1($feed_data) != $cached_feed_data_hash) { |
| | 262 | |
| | 263 | if ($debug_enabled) { |
| | 264 | _debug("update_rss_feed: saving $cache_filename"); |
| | 265 | } |
| | 266 | |
| | 267 | @file_put_contents($cache_filename, $feed_data); |
| | 268 | } |
| | 269 | } |
| | 270 | |
| | 271 | $pluginhost = new PluginHost($link); |
| | 272 | $pluginhost->set_debug($debug_enabled); |
| | 273 | $user_plugins = get_pref($link, "_ENABLED_PLUGINS", $owner_uid); |
| | 274 | |
| | 275 | $pluginhost->load(PLUGINS, $pluginhost::KIND_ALL); |
| | 276 | $pluginhost->load($user_plugins, $pluginhost::KIND_USER, $owner_uid); |
| | 277 | $pluginhost->load_data(); |
| | 278 | |
| | 279 | foreach ($pluginhost->get_hooks($pluginhost::HOOK_FEED_FETCHED) as $plugin) { |
| | 280 | $feed_data = $plugin->hook_feed_fetched($feed_data); |
| | 281 | } |
| | 282 | |
| | 283 | if ($debug_enabled) { |
| | 284 | _debug("update_rss_feed: fetch done, parsing..."); |
| | 285 | } |
| | 286 | |
| | 287 | $rss = new SimplePie(); |
| | 288 | $rss->set_sanitize_class("SanitizeDummy"); |
| | 289 | // simplepie ignores the above and creates default sanitizer anyway, |
| | 290 | // so let's override it... |
| | 291 | $rss->sanitize = new SanitizeDummy(); |
| | 292 | $rss->set_output_encoding('UTF-8'); |
| | 293 | $rss->set_raw_data($feed_data); |
| | 294 | $rss->enable_cache(false); |
| | 295 | |
| | 296 | @$rss->init(); |
| | 297 | |
| | 298 | // print_r($rss); |
| | 299 | |
| | 300 | $feed = db_escape_string($link, $feed); |
| | 301 | |
| | 302 | if (!$rss->error()) { |
| | 303 | |
| | 304 | // We use local pluginhost here because we need to load different per-user feed plugins |
| | 305 | $pluginhost->run_hooks($pluginhost::HOOK_FEED_PARSED, "hook_feed_parsed", $rss); |
| | 306 | |
| | 307 | if ($debug_enabled) { |
| | 308 | _debug("update_rss_feed: processing feed data..."); |
| | 309 | } |
| | 310 | |
| | 311 | // db_query($link, "BEGIN"); |
| | 312 | |
| | 313 | if (DB_TYPE == "pgsql") { |
| | 314 | $favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'"; |
| | 315 | } else { |
| | 316 | $favicon_interval_qpart = "favicon_last_checked < DATE_SUB(NOW(), INTERVAL 12 HOUR)"; |
| | 317 | } |
| | 318 | |
| | 319 | $result = db_query($link, "SELECT title,site_url,owner_uid, |
| | 320 | (favicon_last_checked IS NULL OR $favicon_interval_qpart) AS |
| | 321 | favicon_needs_check |
| | 322 | FROM ttrss_feeds WHERE id = '$feed'"); |
| | 323 | |
| | 324 | $registered_title = db_fetch_result($result, 0, "title"); |
| | 325 | $orig_site_url = db_fetch_result($result, 0, "site_url"); |
| | 326 | $favicon_needs_check = sql_bool_to_bool(db_fetch_result($result, 0, |
| | 327 | "favicon_needs_check")); |
| | 328 | |
| | 329 | $owner_uid = db_fetch_result($result, 0, "owner_uid"); |
| | 330 | |
| | 331 | $site_url = db_escape_string($link, mb_substr(rewrite_relative_url($fetch_url, $rss->get_link()), 0, 245)); |
| | 332 | |
| | 333 | if ($debug_enabled) { |
| | 334 | _debug("update_rss_feed: checking favicon..."); |
| | 335 | } |
| | 336 | |
| | 337 | if ($favicon_needs_check) { |
| | 338 | check_feed_favicon($site_url, $feed, $link); |
| | 339 | |
| | 340 | db_query($link, "UPDATE ttrss_feeds SET favicon_last_checked = NOW() |
| | 341 | WHERE id = '$feed'"); |
| | 342 | } |
| | 343 | |
| | 344 | if (!$registered_title || $registered_title == "[Unknown]") { |
| | 345 | |
| | 346 | $feed_title = db_escape_string($link, $rss->get_title()); |
| | 347 | |
| | 348 | if ($debug_enabled) { |
| | 349 | _debug("update_rss_feed: registering title: $feed_title"); |
| | 350 | } |
| | 351 | |
| | 352 | db_query($link, "UPDATE ttrss_feeds SET |
| | 353 | title = '$feed_title' WHERE id = '$feed'"); |
| | 354 | } |
| | 355 | |
| | 356 | if ($site_url && $orig_site_url != $site_url) { |
| | 357 | db_query($link, "UPDATE ttrss_feeds SET |
| | 358 | site_url = '$site_url' WHERE id = '$feed'"); |
| | 359 | } |
| | 360 | |
| | 361 | if ($debug_enabled) { |
| | 362 | _debug("update_rss_feed: loading filters & labels..."); |
| | 363 | } |
| | 364 | |
| | 365 | $filters = load_filters($link, $feed, $owner_uid); |
| | 366 | $labels = get_all_labels($link, $owner_uid); |
| | 367 | |
| | 368 | if ($debug_enabled) { |
| | 369 | //print_r($filters); |
| | 370 | _debug("update_rss_feed: " . count($filters) . " filters loaded."); |
| | 371 | } |
| | 372 | |
| | 373 | $items = $rss->get_items(); |
| | 374 | |
| | 375 | if (!is_array($items)) { |
| | 376 | if ($debug_enabled) { |
| | 377 | _debug("update_rss_feed: no articles found."); |
| | 378 | } |
| | 379 | |
| | 380 | db_query($link, "UPDATE ttrss_feeds |
| | 381 | SET last_updated = NOW(), last_error = '' WHERE id = '$feed'"); |
| | 382 | |
| | 383 | return; // no articles |
| | 384 | } |
| | 385 | |
| | 386 | if ($pubsub_state != 2 && PUBSUBHUBBUB_ENABLED) { |
| | 387 | |
| | 388 | if ($debug_enabled) _debug("update_rss_feed: checking for PUSH hub..."); |
| | 389 | |
| | 390 | $feed_hub_url = false; |
| | 391 | |
| | 392 | $links = $rss->get_links('hub'); |
| | 393 | |
| | 394 | if ($links && is_array($links)) { |
| | 395 | foreach ($links as $l) { |
| | 396 | $feed_hub_url = $l; |
| | 397 | break; |
| | 398 | } |
| | 399 | } |
| | 400 | |
| | 401 | if ($debug_enabled) _debug("update_rss_feed: feed hub url: $feed_hub_url"); |
| | 402 | |
| | 403 | if ($feed_hub_url && function_exists('curl_init') && |
| | 404 | !ini_get("open_basedir")) { |
| | 405 | |
| | 406 | require_once 'lib/pubsubhubbub/subscriber.php'; |
| | 407 | |
| | 408 | $callback_url = get_self_url_prefix() . |
| | 409 | "/public.php?op=pubsub&id=$feed"; |
| | 410 | |
| | 411 | $s = new Subscriber($feed_hub_url, $callback_url); |
| | 412 | |
| | 413 | $rc = $s->subscribe($fetch_url); |
| | 414 | |
| | 415 | if ($debug_enabled) |
| | 416 | _debug("update_rss_feed: feed hub url found, subscribe request sent."); |
| | 417 | |
| | 418 | db_query($link, "UPDATE ttrss_feeds SET pubsub_state = 1 |
| | 419 | WHERE id = '$feed'"); |
| | 420 | } |
| | 421 | } |
| | 422 | |
| | 423 | if ($debug_enabled) { |
| | 424 | _debug("update_rss_feed: processing articles..."); |
| | 425 | } |
| | 426 | |
| | 427 | foreach ($items as $item) { |
| | 428 | if ($_REQUEST['xdebug'] == 3) { |
| | 429 | print_r($item); |
| | 430 | } |
| | 431 | |
| | 432 | $entry_guid = $item->get_id(); |
| | 433 | if (!$entry_guid) $entry_guid = $item->get_link(); |
| | 434 | if (!$entry_guid) $entry_guid = make_guid_from_title($item->get_title()); |
| | 435 | |
| | 436 | if ($debug_enabled) { |
| | 437 | _debug("update_rss_feed: guid $entry_guid"); |
| | 438 | } |
| | 439 | |
| | 440 | if (!$entry_guid) continue; |
| | 441 | |
| | 442 | $entry_guid = "$owner_uid,$entry_guid"; |
| | 443 | |
| | 444 | $entry_timestamp = ""; |
| | 445 | |
| | 446 | $entry_timestamp = strtotime($item->get_date()); |
| | 447 | |
| | 448 | if ($entry_timestamp == -1 || !$entry_timestamp || $entry_timestamp > time()) { |
| | 449 | $entry_timestamp = time(); |
| | 450 | $no_orig_date = 'true'; |
| | 451 | } else { |
| | 452 | $no_orig_date = 'false'; |
| | 453 | } |
| | 454 | |
| | 455 | $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); |
| | 456 | |
| | 457 | if ($debug_enabled) { |
| | 458 | _debug("update_rss_feed: date $entry_timestamp [$entry_timestamp_fmt]"); |
| | 459 | } |
| | 460 | |
| | 461 | $entry_title = $item->get_title(); |
| | 462 | |
| | 463 | $entry_link = rewrite_relative_url($site_url, $item->get_link()); |
| | 464 | |
| | 465 | if ($debug_enabled) { |
| | 466 | _debug("update_rss_feed: title $entry_title"); |
| | 467 | _debug("update_rss_feed: link $entry_link"); |
| | 468 | } |
| | 469 | |
| | 470 | if (!$entry_title) $entry_title = date("Y-m-d H:i:s", $entry_timestamp);; |
| | 471 | |
| | 472 | $entry_content = $item->get_content(); |
| | 473 | if (!$entry_content) $entry_content = $item->get_description(); |
| | 474 | |
| | 475 | if ($_REQUEST["xdebug"] == 2) { |
| | 476 | print "update_rss_feed: content: "; |
| | 477 | print $entry_content; |
| | 478 | print "\n"; |
| | 479 | } |
| | 480 | |
| | 481 | $entry_comments = $item->data["comments"]; |
| | 482 | |
| | 483 | if ($item->get_author()) { |
| | 484 | $entry_author_item = $item->get_author(); |
| | 485 | $entry_author = $entry_author_item->get_name(); |
| | 486 | if (!$entry_author) $entry_author = $entry_author_item->get_email(); |
| | 487 | |
| | 488 | $entry_author = db_escape_string($link, $entry_author); |
| | 489 | } |
| | 490 | |
| | 491 | $entry_guid = db_escape_string($link, mb_substr($entry_guid, 0, 245)); |
| | 492 | |
| | 493 | $entry_comments = db_escape_string($link, mb_substr($entry_comments, 0, 245)); |
| | 494 | $entry_author = db_escape_string($link, mb_substr($entry_author, 0, 245)); |
| | 495 | |
| | 496 | $num_comments = $item->get_item_tags('http://purl.org/rss/1.0/modules/slash/', 'comments'); |
| | 497 | |
| | 498 | if (is_array($num_comments) && is_array($num_comments[0])) { |
| | 499 | $num_comments = (int) $num_comments[0]["data"]; |
| | 500 | } else { |
| | 501 | $num_comments = 0; |
| | 502 | } |
| | 503 | |
| | 504 | if ($debug_enabled) { |
| | 505 | _debug("update_rss_feed: num_comments: $num_comments"); |
| | 506 | _debug("update_rss_feed: looking for tags [1]..."); |
| | 507 | } |
| | 508 | |
| | 509 | // parse <category> entries into tags |
| | 510 | |
| | 511 | $additional_tags = array(); |
| | 512 | |
| | 513 | $additional_tags_src = $item->get_categories(); |
| | 514 | |
| | 515 | if (is_array($additional_tags_src)) { |
| | 516 | foreach ($additional_tags_src as $tobj) { |
| | 517 | array_push($additional_tags, $tobj->get_term()); |
| | 518 | } |
| | 519 | } |
| | 520 | |
| | 521 | if ($debug_enabled) { |
| | 522 | _debug("update_rss_feed: category tags:"); |
| | 523 | print_r($additional_tags); |
| | 524 | } |
| | 525 | |
| | 526 | if ($debug_enabled) { |
| | 527 | _debug("update_rss_feed: looking for tags [2]..."); |
| | 528 | } |
| | 529 | |
| | 530 | $entry_tags = array_unique($additional_tags); |
| | 531 | |
| | 532 | for ($i = 0; $i < count($entry_tags); $i++) |
| | 533 | $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8'); |
| | 534 | |
| | 535 | if ($debug_enabled) { |
| | 536 | //_debug("update_rss_feed: unfiltered tags found:"); |
| | 537 | //print_r($entry_tags); |
| | 538 | } |
| | 539 | |
| | 540 | if ($debug_enabled) { |
| | 541 | _debug("update_rss_feed: done collecting data."); |
| | 542 | } |
| | 543 | |
| | 544 | // TODO: less memory-hungry implementation |
| | 545 | |
| | 546 | if ($debug_enabled) { |
| | 547 | _debug("update_rss_feed: applying plugin filters.."); |
| | 548 | } |
| | 549 | |
| | 550 | // FIXME not sure if owner_uid is a good idea here, we may have a base entry without user entry (?) |
| | 551 | $result = db_query($link, "SELECT plugin_data,title,content,link,tag_cache,author FROM ttrss_entries, ttrss_user_entries |
| | 552 | WHERE ref_id = id AND guid = '".db_escape_string($link, $entry_guid)."' AND owner_uid = $owner_uid"); |
| | 553 | |
| | 554 | if (db_num_rows($result) != 0) { |
| | 555 | $entry_plugin_data = db_fetch_result($result, 0, "plugin_data"); |
| | 556 | $stored_article = array("title" => db_fetch_result($result, 0, "title"), |
| | 557 | "content" => db_fetch_result($result, 0, "content"), |
| | 558 | "link" => db_fetch_result($result, 0, "link"), |
| | 559 | "tags" => explode(",", db_fetch_result($result, 0, "tag_cache")), |
| | 560 | "author" => db_fetch_result($result, 0, "author")); |
| | 561 | } else { |
| | 562 | $entry_plugin_data = ""; |
| | 563 | $stored_article = array(); |
| | 564 | } |
| | 565 | |
| | 566 | $article = array("owner_uid" => $owner_uid, // read only |
| | 567 | "guid" => $entry_guid, // read only |
| | 568 | "title" => $entry_title, |
| | 569 | "content" => $entry_content, |
| | 570 | "link" => $entry_link, |
| | 571 | "tags" => $entry_tags, |
| | 572 | "plugin_data" => $entry_plugin_data, |
| | 573 | "author" => $entry_author, |
| | 574 | "stored" => $stored_article); |
| | 575 | |
| | 576 | foreach ($pluginhost->get_hooks($pluginhost::HOOK_ARTICLE_FILTER) as $plugin) { |
| | 577 | $article = $plugin->hook_article_filter($article); |
| | 578 | } |
| | 579 | |
| | 580 | $entry_tags = $article["tags"]; |
| | 581 | $entry_guid = db_escape_string($link, $entry_guid); |
| | 582 | $entry_title = db_escape_string($link, $article["title"]); |
| | 583 | $entry_author = db_escape_string($link, $article["author"]); |
| | 584 | $entry_link = db_escape_string($link, $article["link"]); |
| | 585 | $entry_plugin_data = db_escape_string($link, $article["plugin_data"]); |
| | 586 | $entry_content = $article["content"]; // escaped below |
| | 587 | |
| | 588 | |
| | 589 | if ($debug_enabled) { |
| | 590 | _debug("update_rss_feed: plugin data: $entry_plugin_data"); |
| | 591 | } |
| | 592 | |
| | 593 | if ($cache_images && is_writable(CACHE_DIR . '/images')) |
| | 594 | cache_images($entry_content, $site_url, $debug_enabled); |
| | 595 | |
| | 596 | $entry_content = db_escape_string($link, $entry_content, false); |
| | 597 | |
| | 598 | $content_hash = "SHA1:" . sha1($entry_content); |
| | 599 | |
| | 600 | db_query($link, "BEGIN"); |
| | 601 | |
| | 602 | $result = db_query($link, "SELECT id FROM ttrss_entries |
| | 603 | WHERE guid = '$entry_guid'"); |
| | 604 | |
| | 605 | if (db_num_rows($result) == 0) { |
| | 606 | |
| | 607 | if ($debug_enabled) { |
| | 608 | _debug("update_rss_feed: base guid [$entry_guid] not found"); |
| | 609 | } |
| | 610 | |
| | 611 | // base post entry does not exist, create it |
| | 612 | |
| | 613 | $result = db_query($link, |
| | 614 | "INSERT INTO ttrss_entries |
| | 615 | (title, |
| | 616 | guid, |
| | 617 | link, |
| | 618 | updated, |
| | 619 | content, |
| | 620 | content_hash, |
| | 621 | cached_content, |
| | 622 | no_orig_date, |
| | 623 | date_updated, |
| | 624 | date_entered, |
| | 625 | comments, |
| | 626 | num_comments, |
| | 627 | plugin_data, |
| | 628 | author) |
| | 629 | VALUES |
| | 630 | ('$entry_title', |
| | 631 | '$entry_guid', |
| | 632 | '$entry_link', |
| | 633 | '$entry_timestamp_fmt', |
| | 634 | '$entry_content', |
| | 635 | '$content_hash', |
| | 636 | '', |
| | 637 | $no_orig_date, |
| | 638 | NOW(), |
| | 639 | '$date_feed_processed', |
| | 640 | '$entry_comments', |
| | 641 | '$num_comments', |
| | 642 | '$entry_plugin_data', |
| | 643 | '$entry_author')"); |
| | 644 | |
| | 645 | $article_labels = array(); |
| | 646 | |
| | 647 | } else { |
| | 648 | // we keep encountering the entry in feeds, so we need to |
| | 649 | // update date_updated column so that we don't get horrible |
| | 650 | // dupes when the entry gets purged and reinserted again e.g. |
| | 651 | // in the case of SLOW SLOW OMG SLOW updating feeds |
| | 652 | |
| | 653 | $base_entry_id = db_fetch_result($result, 0, "id"); |
| | 654 | |
| | 655 | db_query($link, "UPDATE ttrss_entries SET date_updated = NOW() |
| | 656 | WHERE id = '$base_entry_id'"); |
| | 657 | |
| | 658 | $article_labels = get_article_labels($link, $base_entry_id, $owner_uid); |
| | 659 | } |
| | 660 | |
| | 661 | // now it should exist, if not - bad luck then |
| | 662 | |
| | 663 | $result = db_query($link, "SELECT |
| | 664 | id,content_hash,no_orig_date,title,plugin_data, |
| | 665 | ".SUBSTRING_FOR_DATE."(date_updated,1,19) as date_updated, |
| | 666 | ".SUBSTRING_FOR_DATE."(updated,1,19) as updated, |
| | 667 | num_comments |
| | 668 | FROM |
| | 669 | ttrss_entries |
| | 670 | WHERE guid = '$entry_guid'"); |
| | 671 | |
| | 672 | $entry_ref_id = 0; |
| | 673 | $entry_int_id = 0; |
| | 674 | |
| | 675 | if (db_num_rows($result) == 1) { |
| | 676 | |
| | 677 | if ($debug_enabled) { |
| | 678 | _debug("update_rss_feed: base guid [$entry_guid] found, checking for user record"); |
| | 679 | } |
| | 680 | |
| | 681 | // this will be used below in update handler |
| | 682 | $orig_content_hash = db_fetch_result($result, 0, "content_hash"); |
| | 683 | $orig_title = db_fetch_result($result, 0, "title"); |
| | 684 | $orig_num_comments = db_fetch_result($result, 0, "num_comments"); |
| | 685 | $orig_date_updated = strtotime(db_fetch_result($result, |
| | 686 | 0, "date_updated")); |
| | 687 | $orig_plugin_data = db_fetch_result($result, 0, "plugin_data"); |
| | 688 | |
| | 689 | $ref_id = db_fetch_result($result, 0, "id"); |
| | 690 | $entry_ref_id = $ref_id; |
| | 691 | |
| | 692 | // check for user post link to main table |
| | 693 | |
| | 694 | // do we allow duplicate posts with same GUID in different feeds? |
| | 695 | if (get_pref($link, "ALLOW_DUPLICATE_POSTS", $owner_uid, false)) { |
| | 696 | $dupcheck_qpart = "AND (feed_id = '$feed' OR feed_id IS NULL)"; |
| | 697 | } else { |
| | 698 | $dupcheck_qpart = ""; |
| | 699 | } |
| | 700 | |
| | 701 | /* Collect article tags here so we could filter by them: */ |
| | 702 | |
| | 703 | $article_filters = get_article_filters($filters, $entry_title, |
| | 704 | $entry_content, $entry_link, $entry_timestamp, $entry_author, |
| | 705 | $entry_tags); |
| | 706 | |
| | 707 | if ($debug_enabled) { |
| | 708 | _debug("update_rss_feed: article filters: "); |
| | 709 | if (count($article_filters) != 0) { |
| | 710 | print_r($article_filters); |
| | 711 | } |
| | 712 | } |
| | 713 | |
| | 714 | if (find_article_filter($article_filters, "filter")) { |
| | 715 | db_query($link, "COMMIT"); // close transaction in progress |
| | 716 | continue; |
| | 717 | } |
| | 718 | |
| | 719 | $score = calculate_article_score($article_filters); |
| | 720 | |
| | 721 | if ($debug_enabled) { |
| | 722 | _debug("update_rss_feed: initial score: $score"); |
| | 723 | } |
| | 724 | |
| | 725 | $query = "SELECT ref_id, int_id FROM ttrss_user_entries WHERE |
| | 726 | ref_id = '$ref_id' AND owner_uid = '$owner_uid' |
| | 727 | $dupcheck_qpart"; |
| | 728 | |
| | 729 | // if ($_REQUEST["xdebug"]) print "$query\n"; |
| | 730 | |
| | 731 | $result = db_query($link, $query); |
| | 732 | |
| | 733 | // okay it doesn't exist - create user entry |
| | 734 | if (db_num_rows($result) == 0) { |
| | 735 | |
| | 736 | if ($debug_enabled) { |
| | 737 | _debug("update_rss_feed: user record not found, creating..."); |
| | 738 | } |
| | 739 | |
| | 740 | if ($score >= -500 && !find_article_filter($article_filters, 'catchup')) { |
| | 741 | $unread = 'true'; |
| | 742 | $last_read_qpart = 'NULL'; |
| | 743 | } else { |
| | 744 | $unread = 'false'; |
| | 745 | $last_read_qpart = 'NOW()'; |
| | 746 | } |
| | 747 | |
| | 748 | if (find_article_filter($article_filters, 'mark') || $score > 1000) { |
| | 749 | $marked = 'true'; |
| | 750 | } else { |
| | 751 | $marked = 'false'; |
| | 752 | } |
| | 753 | |
| | 754 | if (find_article_filter($article_filters, 'publish')) { |
| | 755 | $published = 'true'; |
| | 756 | } else { |
| | 757 | $published = 'false'; |
| | 758 | } |
| | 759 | |
| | 760 | // N-grams |
| | 761 | |
| | 762 | if (DB_TYPE == "pgsql" and defined('_NGRAM_TITLE_DUPLICATE_THRESHOLD')) { |
| | 763 | |
| | 764 | $result = db_query($link, "SELECT COUNT(*) AS similar FROM |
| | 765 | ttrss_entries,ttrss_user_entries |
| | 766 | WHERE ref_id = id AND updated >= NOW() - INTERVAL '7 day' |
| | 767 | AND similarity(title, '$entry_title') >= "._NGRAM_TITLE_DUPLICATE_THRESHOLD." |
| | 768 | AND owner_uid = $owner_uid"); |
| | 769 | |
| | 770 | $ngram_similar = db_fetch_result($result, 0, "similar"); |
| | 771 | |
| | 772 | if ($debug_enabled) { |
| | 773 | _debug("update_rss_feed: N-gram similar results: $ngram_similar"); |
| | 774 | } |
| | 775 | |
| | 776 | if ($ngram_similar > 0) { |
| | 777 | $unread = 'false'; |
| | 778 | } |
| | 779 | } |
| | 780 | |
| | 781 | $last_marked = ($marked == 'true') ? 'NOW()' : 'NULL'; |
| | 782 | $last_published = ($published == 'true') ? 'NOW()' : 'NULL'; |
| | 783 | |
| | 784 | $result = db_query($link, |
| | 785 | "INSERT INTO ttrss_user_entries |
| | 786 | (ref_id, owner_uid, feed_id, unread, last_read, marked, |
| | 787 | published, score, tag_cache, label_cache, uuid, |
| | 788 | last_marked, last_published) |
| | 789 | VALUES ('$ref_id', '$owner_uid', '$feed', $unread, |
| | 790 | $last_read_qpart, $marked, $published, '$score', '', '', |
| | 791 | '', $last_marked, $last_published)"); |
| | 792 | |
| | 793 | if (PUBSUBHUBBUB_HUB && $published == 'true') { |
| | 794 | $rss_link = get_self_url_prefix() . |
| | 795 | "/public.php?op=rss&id=-2&key=" . |
| | 796 | get_feed_access_key($link, -2, false, $owner_uid); |
| | 797 | |
| | 798 | $p = new Publisher(PUBSUBHUBBUB_HUB); |
| | 799 | |
| | 800 | $pubsub_result = $p->publish_update($rss_link); |
| | 801 | } |
| | 802 | |
| | 803 | $result = db_query($link, |
| | 804 | "SELECT int_id FROM ttrss_user_entries WHERE |
| | 805 | ref_id = '$ref_id' AND owner_uid = '$owner_uid' AND |
| | 806 | feed_id = '$feed' LIMIT 1"); |
| | 807 | |
| | 808 | if (db_num_rows($result) == 1) { |
| | 809 | $entry_int_id = db_fetch_result($result, 0, "int_id"); |
| | 810 | } |
| | 811 | } else { |
| | 812 | if ($debug_enabled) { |
| | 813 | _debug("update_rss_feed: user record FOUND"); |
| | 814 | } |
| | 815 | |
| | 816 | $entry_ref_id = db_fetch_result($result, 0, "ref_id"); |
| | 817 | $entry_int_id = db_fetch_result($result, 0, "int_id"); |
| | 818 | } |
| | 819 | |
| | 820 | if ($debug_enabled) { |
| | 821 | _debug("update_rss_feed: RID: $entry_ref_id, IID: $entry_int_id"); |
| | 822 | } |
| | 823 | |
| | 824 | $post_needs_update = false; |
| | 825 | $update_insignificant = false; |
| | 826 | |
| | 827 | if ($orig_num_comments != $num_comments) { |
| | 828 | $post_needs_update = true; |
| | 829 | $update_insignificant = true; |
| | 830 | } |
| | 831 | |
| | 832 | if ($entry_plugin_data != $orig_plugin_data) { |
| | 833 | $post_needs_update = true; |
| | 834 | $update_insignificant = true; |
| | 835 | } |
| | 836 | |
| | 837 | if ($content_hash != $orig_content_hash) { |
| | 838 | $post_needs_update = true; |
| | 839 | $update_insignificant = false; |
| | 840 | } |
| | 841 | |
| | 842 | if (db_escape_string($link, $orig_title) != $entry_title) { |
| | 843 | $post_needs_update = true; |
| | 844 | $update_insignificant = false; |
| | 845 | } |
| | 846 | |
| | 847 | // if post needs update, update it and mark all user entries |
| | 848 | // linking to this post as updated |
| | 849 | if ($post_needs_update) { |
| | 850 | |
| | 851 | if (defined('DAEMON_EXTENDED_DEBUG')) { |
| | 852 | _debug("update_rss_feed: post $entry_guid needs update..."); |
| | 853 | } |
| | 854 | |
| | 855 | // print "<!-- post $orig_title needs update : $post_needs_update -->"; |
| | 856 | |
| | 857 | db_query($link, "UPDATE ttrss_entries |
| | 858 | SET title = '$entry_title', content = '$entry_content', |
| | 859 | content_hash = '$content_hash', |
| | 860 | updated = '$entry_timestamp_fmt', |
| | 861 | num_comments = '$num_comments', |
| | 862 | plugin_data = '$entry_plugin_data' |
| | 863 | WHERE id = '$ref_id'"); |
| | 864 | |
| | 865 | if (!$update_insignificant) { |
| | 866 | if ($mark_unread_on_update) { |
| | 867 | db_query($link, "UPDATE ttrss_user_entries |
| | 868 | SET last_read = null, unread = true WHERE ref_id = '$ref_id'"); |
| | 869 | } |
| | 870 | } |
| | 871 | } |
| | 872 | } |
| | 873 | |
| | 874 | db_query($link, "COMMIT"); |
| | 875 | |
| | 876 | if ($debug_enabled) { |
| | 877 | _debug("update_rss_feed: assigning labels..."); |
| | 878 | } |
| | 879 | |
| | 880 | assign_article_to_label_filters($link, $entry_ref_id, $article_filters, |
| | 881 | $owner_uid, $article_labels); |
| | 882 | |
| | 883 | if ($debug_enabled) { |
| | 884 | _debug("update_rss_feed: looking for enclosures..."); |
| | 885 | } |
| | 886 | |
| | 887 | // enclosures |
| | 888 | |
| | 889 | $enclosures = array(); |
| | 890 | |
| | 891 | $encs = $item->get_enclosures(); |
| | 892 | |
| | 893 | if (is_array($encs)) { |
| | 894 | foreach ($encs as $e) { |
| | 895 | $e_item = array( |
| | 896 | $e->link, $e->type, $e->length); |
| | 897 | array_push($enclosures, $e_item); |
| | 898 | } |
| | 899 | } |
| | 900 | |
| | 901 | if ($debug_enabled) { |
| | 902 | _debug("update_rss_feed: article enclosures:"); |
| | 903 | print_r($enclosures); |
| | 904 | } |
| | 905 | |
| | 906 | db_query($link, "BEGIN"); |
| | 907 | |
| | 908 | foreach ($enclosures as $enc) { |
| | 909 | $enc_url = db_escape_string($link, $enc[0]); |
| | 910 | $enc_type = db_escape_string($link, $enc[1]); |
| | 911 | $enc_dur = db_escape_string($link, $enc[2]); |
| | 912 | |
| | 913 | $result = db_query($link, "SELECT id FROM ttrss_enclosures |
| | 914 | WHERE content_url = '$enc_url' AND post_id = '$entry_ref_id'"); |
| | 915 | |
| | 916 | if (db_num_rows($result) == 0) { |
| | 917 | db_query($link, "INSERT INTO ttrss_enclosures |
| | 918 | (content_url, content_type, title, duration, post_id) VALUES |
| | 919 | ('$enc_url', '$enc_type', '', '$enc_dur', '$entry_ref_id')"); |
| | 920 | } |
| | 921 | } |
| | 922 | |
| | 923 | db_query($link, "COMMIT"); |
| | 924 | |
| | 925 | // check for manual tags (we have to do it here since they're loaded from filters) |
| | 926 | |
| | 927 | foreach ($article_filters as $f) { |
| | 928 | if ($f["type"] == "tag") { |
| | 929 | |
| | 930 | $manual_tags = trim_array(explode(",", $f["param"])); |
| | 931 | |
| | 932 | foreach ($manual_tags as $tag) { |
| | 933 | if (tag_is_valid($tag)) { |
| | 934 | array_push($entry_tags, $tag); |
| | 935 | } |
| | 936 | } |
| | 937 | } |
| | 938 | } |
| | 939 | |
| | 940 | // Skip boring tags |
| | 941 | |
| | 942 | $boring_tags = trim_array(explode(",", mb_strtolower(get_pref($link, |
| | 943 | 'BLACKLISTED_TAGS', $owner_uid, ''), 'utf-8'))); |
| | 944 | |
| | 945 | $filtered_tags = array(); |
| | 946 | $tags_to_cache = array(); |
| | 947 | |
| | 948 | if ($entry_tags && is_array($entry_tags)) { |
| | 949 | foreach ($entry_tags as $tag) { |
| | 950 | if (array_search($tag, $boring_tags) === false) { |
| | 951 | array_push($filtered_tags, $tag); |
| | 952 | } |
| | 953 | } |
| | 954 | } |
| | 955 | |
| | 956 | $filtered_tags = array_unique($filtered_tags); |
| | 957 | |
| | 958 | if ($debug_enabled) { |
| | 959 | _debug("update_rss_feed: filtered article tags:"); |
| | 960 | print_r($filtered_tags); |
| | 961 | } |
| | 962 | |
| | 963 | // Save article tags in the database |
| | 964 | |
| | 965 | if (count($filtered_tags) > 0) { |
| | 966 | |
| | 967 | db_query($link, "BEGIN"); |
| | 968 | |
| | 969 | foreach ($filtered_tags as $tag) { |
| | 970 | |
| | 971 | $tag = sanitize_tag($tag); |
| | 972 | $tag = db_escape_string($link, $tag); |
| | 973 | |
| | 974 | if (!tag_is_valid($tag)) continue; |
| | 975 | |
| | 976 | $result = db_query($link, "SELECT id FROM ttrss_tags |
| | 977 | WHERE tag_name = '$tag' AND post_int_id = '$entry_int_id' AND |
| | 978 | owner_uid = '$owner_uid' LIMIT 1"); |
| | 979 | |
| | 980 | if ($result && db_num_rows($result) == 0) { |
| | 981 | |
| | 982 | db_query($link, "INSERT INTO ttrss_tags |
| | 983 | (owner_uid,tag_name,post_int_id) |
| | 984 | VALUES ('$owner_uid','$tag', '$entry_int_id')"); |
| | 985 | } |
| | 986 | |
| | 987 | array_push($tags_to_cache, $tag); |
| | 988 | } |
| | 989 | |
| | 990 | /* update the cache */ |
| | 991 | |
| | 992 | $tags_to_cache = array_unique($tags_to_cache); |
| | 993 | |
| | 994 | $tags_str = db_escape_string($link, join(",", $tags_to_cache)); |
| | 995 | |
| | 996 | db_query($link, "UPDATE ttrss_user_entries |
| | 997 | SET tag_cache = '$tags_str' WHERE ref_id = '$entry_ref_id' |
| | 998 | AND owner_uid = $owner_uid"); |
| | 999 | |
| | 1000 | db_query($link, "COMMIT"); |
| | 1001 | } |
| | 1002 | |
| | 1003 | if (get_pref($link, "AUTO_ASSIGN_LABELS", $owner_uid, false)) { |
| | 1004 | if ($debug_enabled) { |
| | 1005 | _debug("update_rss_feed: auto-assigning labels..."); |
| | 1006 | } |
| | 1007 | |
| | 1008 | foreach ($labels as $label) { |
| | 1009 | $caption = preg_quote($label["caption"]); |
| | 1010 | |
| | 1011 | if ($caption && preg_match("/\b$caption\b/i", "$tags_str " . strip_tags($entry_content) . " $entry_title")) { |
| | 1012 | if (!labels_contains_caption($article_labels, $caption)) { |
| | 1013 | label_add_article($link, $entry_ref_id, $caption, $owner_uid); |
| | 1014 | } |
| | 1015 | } |
| | 1016 | } |
| | 1017 | } |
| | 1018 | |
| | 1019 | if ($debug_enabled) { |
| | 1020 | _debug("update_rss_feed: article processed"); |
| | 1021 | } |
| | 1022 | } |
| | 1023 | |
| | 1024 | if (!$last_updated) { |
| | 1025 | if ($debug_enabled) { |
| | 1026 | _debug("update_rss_feed: new feed, catching it up..."); |
| | 1027 | } |
| | 1028 | catchup_feed($link, $feed, false, $owner_uid); |
| | 1029 | } |
| | 1030 | |
| | 1031 | if ($debug_enabled) { |
| | 1032 | _debug("purging feed..."); |
| | 1033 | } |
| | 1034 | |
| | 1035 | purge_feed($link, $feed, 0, $debug_enabled); |
| | 1036 | |
| | 1037 | db_query($link, "UPDATE ttrss_feeds |
| | 1038 | SET last_updated = NOW(), last_error = '' WHERE id = '$feed'"); |
| | 1039 | |
| | 1040 | // db_query($link, "COMMIT"); |
| | 1041 | |
| | 1042 | } else { |
| | 1043 | |
| | 1044 | $error_msg = db_escape_string($link, mb_substr($rss->error(), 0, 245)); |
| | 1045 | |
| | 1046 | if ($debug_enabled) { |
| | 1047 | _debug("update_rss_feed: error fetching feed: $error_msg"); |
| | 1048 | } |
| | 1049 | |
| | 1050 | db_query($link, |
| | 1051 | "UPDATE ttrss_feeds SET last_error = '$error_msg', |
| | 1052 | last_updated = NOW() WHERE id = '$feed'"); |
| | 1053 | } |
| | 1054 | |
| | 1055 | unset($rss); |
| | 1056 | |
| | 1057 | if ($debug_enabled) { |
| | 1058 | _debug("update_rss_feed: done"); |
| | 1059 | } |
| | 1060 | |
| | 1061 | } |
| | 1062 | |
| | 1063 | function cache_images($html, $site_url, $debug) { |
| | 1064 | $cache_dir = CACHE_DIR . "/images"; |
| | 1065 | |
| | 1066 | libxml_use_internal_errors(true); |
| | 1067 | |
| | 1068 | $charset_hack = '<head> |
| | 1069 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> |
| | 1070 | </head>'; |
| | 1071 | |
| | 1072 | $doc = new DOMDocument(); |
| | 1073 | $doc->loadHTML($charset_hack . $html); |
| | 1074 | $xpath = new DOMXPath($doc); |
| | 1075 | |
| | 1076 | $entries = $xpath->query('(//img[@src])'); |
| | 1077 | |
| | 1078 | foreach ($entries as $entry) { |
| | 1079 | if ($entry->hasAttribute('src')) { |
| | 1080 | $src = rewrite_relative_url($site_url, $entry->getAttribute('src')); |
| | 1081 | |
| | 1082 | $local_filename = CACHE_DIR . "/images/" . sha1($src) . ".png"; |
| | 1083 | |
| | 1084 | if ($debug) _debug("cache_images: downloading: $src to $local_filename"); |
| | 1085 | |
| | 1086 | if (!file_exists($local_filename)) { |
| | 1087 | $file_content = fetch_file_contents($src); |
| | 1088 | |
| | 1089 | if ($file_content && strlen($file_content) > 1024) { |
| | 1090 | file_put_contents($local_filename, $file_content); |
| | 1091 | } |
| | 1092 | } |
| | 1093 | |
| | 1094 | if (file_exists($local_filename)) { |
| | 1095 | $entry->setAttribute('src', SELF_URL_PATH . '/image.php?url=' . |
| | 1096 | base64_encode($src)); |
| | 1097 | } |
| | 1098 | } |
| | 1099 | } |
| | 1100 | |
| | 1101 | $node = $doc->getElementsByTagName('body')->item(0); |
| | 1102 | |
| | 1103 | return $doc->saveXML($node); |
| | 1104 | } |
| | 1105 | |
| | 1106 | function expire_lock_files($debug) { |
| | 1107 | if ($debug) _debug("Removing old lock files..."); |
| | 1108 | |
| | 1109 | $num_deleted = 0; |
| | 1110 | |
| | 1111 | if (is_writable(LOCK_DIRECTORY)) { |
| | 1112 | $files = glob(LOCK_DIRECTORY . "/*.lock"); |
| | 1113 | |
| | 1114 | if ($files) { |
| | 1115 | foreach ($files as $file) { |
| | 1116 | if (!file_is_locked($file) && time() - filemtime($file) > 86400*2) { |
| | 1117 | unlink($file); |
| | 1118 | ++$num_deleted; |
| | 1119 | } |
| | 1120 | } |
| | 1121 | } |
| | 1122 | } |
| | 1123 | |
| | 1124 | if ($debug) _debug("Removed $num_deleted files."); |
| | 1125 | } |
| | 1126 | |
| | 1127 | function expire_cached_files($debug) { |
| | 1128 | foreach (array("simplepie", "images", "export") as $dir) { |
| | 1129 | $cache_dir = CACHE_DIR . "/$dir"; |
| | 1130 | |
| | 1131 | if ($debug) _debug("Expiring $cache_dir"); |
| | 1132 | |
| | 1133 | $num_deleted = 0; |
| | 1134 | |
| | 1135 | if (is_writable($cache_dir)) { |
| | 1136 | $files = glob("$cache_dir/*"); |
| | 1137 | |
| | 1138 | if ($files) { |
| | 1139 | foreach ($files as $file) { |
| | 1140 | if (time() - filemtime($file) > 86400*7) { |
| | 1141 | unlink($file); |
| | 1142 | |
| | 1143 | ++$num_deleted; |
| | 1144 | } |
| | 1145 | } |
| | 1146 | } |
| | 1147 | } |
| | 1148 | |
| | 1149 | if ($debug) _debug("Removed $num_deleted files."); |
| | 1150 | } |
| | 1151 | } |
| | 1152 | |
| | 1153 | /** |
| | 1154 | * Source: http://www.php.net/manual/en/function.parse-url.php#104527 |
| | 1155 | * Returns the url query as associative array |
| | 1156 | * |
| | 1157 | * @param string query |
| | 1158 | * @return array params |
| | 1159 | */ |
| | 1160 | function convertUrlQuery($query) { |
| | 1161 | $queryParts = explode('&', $query); |
| | 1162 | |
| | 1163 | $params = array(); |
| | 1164 | |
| | 1165 | foreach ($queryParts as $param) { |
| | 1166 | $item = explode('=', $param); |
| | 1167 | $params[$item[0]] = $item[1]; |
| | 1168 | } |
| | 1169 | |
| | 1170 | return $params; |
| | 1171 | } |
| | 1172 | |
| | 1173 | function get_article_filters($filters, $title, $content, $link, $timestamp, $author, $tags) { |
| | 1174 | $matches = array(); |
| | 1175 | |
| | 1176 | foreach ($filters as $filter) { |
| | 1177 | $match_any_rule = $filter["match_any_rule"]; |
| | 1178 | $inverse = $filter["inverse"]; |
| | 1179 | $filter_match = false; |
| | 1180 | |
| | 1181 | foreach ($filter["rules"] as $rule) { |
| | 1182 | $match = false; |
| | 1183 | $reg_exp = $rule["reg_exp"]; |
| | 1184 | $rule_inverse = $rule["inverse"]; |
| | 1185 | |
| | 1186 | if (!$reg_exp) |
| | 1187 | continue; |
| | 1188 | |
| | 1189 | switch ($rule["type"]) { |
| | 1190 | case "title": |
| | 1191 | $match = @preg_match("/$reg_exp/i", $title); |
| | 1192 | break; |
| | 1193 | case "content": |
| | 1194 | // we don't need to deal with multiline regexps |
| | 1195 | $content = preg_replace("/[\r\n\t]/", "", $content); |
| | 1196 | |
| | 1197 | $match = @preg_match("/$reg_exp/i", $content); |
| | 1198 | break; |
| | 1199 | case "both": |
| | 1200 | // we don't need to deal with multiline regexps |
| | 1201 | $content = preg_replace("/[\r\n\t]/", "", $content); |
| | 1202 | |
| | 1203 | $match = (@preg_match("/$reg_exp/i", $title) || @preg_match("/$reg_exp/i", $content)); |
| | 1204 | break; |
| | 1205 | case "link": |
| | 1206 | $match = @preg_match("/$reg_exp/i", $link); |
| | 1207 | break; |
| | 1208 | case "author": |
| | 1209 | $match = @preg_match("/$reg_exp/i", $author); |
| | 1210 | break; |
| | 1211 | case "tag": |
| | 1212 | $tag_string = join(",", $tags); |
| | 1213 | $match = @preg_match("/$reg_exp/i", $tag_string); |
| | 1214 | break; |
| | 1215 | } |
| | 1216 | |
| | 1217 | if ($rule_inverse) $match = !$match; |
| | 1218 | |
| | 1219 | if ($match_any_rule) { |
| | 1220 | if ($match) { |
| | 1221 | $filter_match = true; |
| | 1222 | break; |
| | 1223 | } |
| | 1224 | } else { |
| | 1225 | $filter_match = $match; |
| | 1226 | if (!$match) { |
| | 1227 | break; |
| | 1228 | } |
| | 1229 | } |
| | 1230 | } |
| | 1231 | |
| | 1232 | if ($inverse) $filter_match = !$filter_match; |
| | 1233 | |
| | 1234 | if ($filter_match) { |
| | 1235 | foreach ($filter["actions"] AS $action) { |
| | 1236 | array_push($matches, $action); |
| | 1237 | } |
| | 1238 | } |
| | 1239 | } |
| | 1240 | |
| | 1241 | return $matches; |
| | 1242 | } |
| | 1243 | |
| | 1244 | function find_article_filter($filters, $filter_name) { |
| | 1245 | foreach ($filters as $f) { |
| | 1246 | if ($f["type"] == $filter_name) { |
| | 1247 | return $f; |
| | 1248 | }; |
| | 1249 | } |
| | 1250 | return false; |
| | 1251 | } |
| | 1252 | |
| | 1253 | function find_article_filters($filters, $filter_name) { |
| | 1254 | $results = array(); |
| | 1255 | |
| | 1256 | foreach ($filters as $f) { |
| | 1257 | if ($f["type"] == $filter_name) { |
| | 1258 | array_push($results, $f); |
| | 1259 | }; |
| | 1260 | } |
| | 1261 | return $results; |
| | 1262 | } |
| | 1263 | |
| | 1264 | function calculate_article_score($filters) { |
| | 1265 | $score = 0; |
| | 1266 | |
| | 1267 | foreach ($filters as $f) { |
| | 1268 | if ($f["type"] == "score") { |
| | 1269 | $score += $f["param"]; |
| | 1270 | }; |
| | 1271 | } |
| | 1272 | return $score; |
| | 1273 | } |
| | 1274 | |
| | 1275 | function labels_contains_caption($labels, $caption) { |
| | 1276 | foreach ($labels as $label) { |
| | 1277 | if ($label[1] == $caption) { |
| | 1278 | return true; |
| | 1279 | } |
| | 1280 | } |
| | 1281 | |
| | 1282 | return false; |
| | 1283 | } |
| | 1284 | |
| | 1285 | function assign_article_to_label_filters($link, $id, $filters, $owner_uid, $article_labels) { |
| | 1286 | foreach ($filters as $f) { |
| | 1287 | if ($f["type"] == "label") { |
| | 1288 | if (!labels_contains_caption($article_labels, $f["param"])) { |
| | 1289 | label_add_article($link, $id, $f["param"], $owner_uid); |
| | 1290 | } |
| | 1291 | } |
| | 1292 | } |
| | 1293 | } |
| | 1294 | |
| | 1295 | function make_guid_from_title($title) { |
| | 1296 | return preg_replace("/[ \"\',.:;]/", "-", |
| | 1297 | mb_strtolower(strip_tags($title), 'utf-8')); |
| | 1298 | } |
| | 1299 | |
| | 1300 | |
| | 1301 | ?> |
git.wh0rd.org / tt-rss.git / blame_incremental