]> git.wh0rd.org - tt-rss.git/blame - include/rssfuncs.php
ci: fix typo
[tt-rss.git] / include / rssfuncs.php
CommitLineData
2c08214a 1<?php
fce451a4
AD
2 define_default('DAEMON_UPDATE_LOGIN_LIMIT', 30);
3 define_default('DAEMON_FEED_LIMIT', 500);
e2cf81e2 4 define_default('DAEMON_SLEEP_INTERVAL', 120);
41bead9b 5 define_default('_MIN_CACHE_FILE_SIZE', 1024);
09e8bdfd 6
eb16bd9f 7 function calculate_article_hash($article, $pluginhost) {
af244f92
AD
8 $tmp = "";
9
10 foreach ($article as $k => $v) {
11 if ($k != "feed" && isset($v)) {
24e6ff5d
AD
12 $x = strip_tags(is_array($v) ? implode(",", $v) : $v);
13
14 //_debug("$k:" . sha1($x) . ":" . htmlspecialchars($x), true);
15
16 $tmp .= sha1("$k:" . sha1($x));
af244f92
AD
17 }
18 }
19
eb16bd9f 20 return sha1(implode(",", $pluginhost->get_plugin_names()) . $tmp);
b1840673
AD
21 }
22
6322ac79 23 function update_feedbrowser_cache() {
79178062 24
a42c55f0 25 $result = db_query("SELECT feed_url, site_url, title, COUNT(id) AS subscribers
45378752
LD
26 FROM ttrss_feeds WHERE feed_url NOT IN (SELECT feed_url FROM ttrss_feeds
27 WHERE private IS true OR auth_login != '' OR auth_pass != '' OR feed_url LIKE '%:%@%/%')
79178062
AD
28 GROUP BY feed_url, site_url, title ORDER BY subscribers DESC LIMIT 1000");
29
a42c55f0 30 db_query("BEGIN");
79178062 31
a42c55f0 32 db_query("DELETE FROM ttrss_feedbrowser_cache");
79178062
AD
33
34 $count = 0;
35
36 while ($line = db_fetch_assoc($result)) {
a42c55f0
AD
37 $subscribers = db_escape_string($line["subscribers"]);
38 $feed_url = db_escape_string($line["feed_url"]);
39 $title = db_escape_string($line["title"]);
40 $site_url = db_escape_string($line["site_url"]);
79178062 41
a42c55f0 42 $tmp_result = db_query("SELECT subscribers FROM
79178062
AD
43 ttrss_feedbrowser_cache WHERE feed_url = '$feed_url'");
44
45 if (db_num_rows($tmp_result) == 0) {
46
a42c55f0 47 db_query("INSERT INTO ttrss_feedbrowser_cache
79178062
AD
48 (feed_url, site_url, title, subscribers) VALUES ('$feed_url',
49 '$site_url', '$title', '$subscribers')");
50
51 ++$count;
52
53 }
54
55 }
56
a42c55f0 57 db_query("COMMIT");
79178062
AD
58
59 return $count;
60
61 }
62
7b55001e 63 function update_daemon_common($limit = DAEMON_FEED_LIMIT, $debug = true) {
2c08214a
AD
64 // Process all other feeds using last_updated and interval parameters
65
6322ac79 66 $schema_version = get_schema_version();
857efe49
AD
67
68 if ($schema_version != SCHEMA_VERSION) {
69 die("Schema version is wrong, please upgrade the database.\n");
70 }
71
61c1812f
AD
72 define('PREFS_NO_CACHE', true);
73
2c08214a 74 // Test if the user has loggued in recently. If not, it does not update its feeds.
09e8bdfd 75 if (!SINGLE_USER_MODE && DAEMON_UPDATE_LOGIN_LIMIT > 0) {
2c08214a
AD
76 if (DB_TYPE == "pgsql") {
77 $login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'";
78 } else {
79 $login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)";
80 }
81 } else {
82 $login_thresh_qpart = "";
83 }
84
537eca40 85 // Test if the feed need a update (update interval exceeded).
2c08214a
AD
86 if (DB_TYPE == "pgsql") {
87 $update_limit_qpart = "AND ((
88 ttrss_feeds.update_interval = 0
ee0542ce 89 AND ttrss_user_prefs.value != '-1'
2c08214a
AD
90 AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_user_prefs.value || ' minutes') AS INTERVAL)
91 ) OR (
92 ttrss_feeds.update_interval > 0
93 AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_feeds.update_interval || ' minutes') AS INTERVAL)
f08426e3
AD
94 ) OR (ttrss_feeds.last_updated IS NULL
95 AND ttrss_user_prefs.value != '-1')
96 OR (last_updated = '1970-01-01 00:00:00'
97 AND ttrss_user_prefs.value != '-1'))";
2c08214a
AD
98 } else {
99 $update_limit_qpart = "AND ((
100 ttrss_feeds.update_interval = 0
ee0542ce 101 AND ttrss_user_prefs.value != '-1'
2c08214a
AD
102 AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL CONVERT(ttrss_user_prefs.value, SIGNED INTEGER) MINUTE)
103 ) OR (
104 ttrss_feeds.update_interval > 0
105 AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ttrss_feeds.update_interval MINUTE)
f08426e3
AD
106 ) OR (ttrss_feeds.last_updated IS NULL
107 AND ttrss_user_prefs.value != '-1')
108 OR (last_updated = '1970-01-01 00:00:00'
109 AND ttrss_user_prefs.value != '-1'))";
2c08214a
AD
110 }
111
112 // Test if feed is currently being updated by another process.
113 if (DB_TYPE == "pgsql") {
566417c4 114 $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '10 minutes')";
2c08214a 115 } else {
566417c4 116 $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 10 MINUTE))";
2c08214a
AD
117 }
118
119 // Test if there is a limit to number of updated feeds
120 $query_limit = "";
121 if($limit) $query_limit = sprintf("LIMIT %d", $limit);
122
98070db0
TK
123 // Update the least recently updated feeds first
124 $query_order = "ORDER BY last_updated";
125 if (DB_TYPE == "pgsql") $query_order .= " NULLS FIRST";
126
fce451a4 127 $query = "SELECT DISTINCT ttrss_feeds.feed_url, ttrss_feeds.last_updated
2c08214a
AD
128 FROM
129 ttrss_feeds, ttrss_users, ttrss_user_prefs
f4ae0f05 130 WHERE
2c08214a 131 ttrss_feeds.owner_uid = ttrss_users.id
f08426e3 132 AND ttrss_user_prefs.profile IS NULL
2c08214a
AD
133 AND ttrss_users.id = ttrss_user_prefs.owner_uid
134 AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL'
135 $login_thresh_qpart $update_limit_qpart
1c4421fc 136 $updstart_thresh_qpart
98070db0 137 $query_order $query_limit";
fce451a4
AD
138
139 // We search for feed needing update.
140 $result = db_query($query);
2c08214a 141
ca5ff2d9 142 if($debug) _debug(sprintf("Scheduled %d feeds to update...", db_num_rows($result)));
2c08214a
AD
143
144 // Here is a little cache magic in order to minimize risk of double feed updates.
145 $feeds_to_update = array();
146 while ($line = db_fetch_assoc($result)) {
a42c55f0 147 array_push($feeds_to_update, db_escape_string($line['feed_url']));
2c08214a
AD
148 }
149
150 // We update the feed last update started date before anything else.
151 // There is no lag due to feed contents downloads
152 // It prevent an other process to update the same feed.
1c4421fc
AD
153
154 if(count($feeds_to_update) > 0) {
155 $feeds_quoted = array();
156
157 foreach ($feeds_to_update as $feed) {
a42c55f0 158 array_push($feeds_quoted, "'" . db_escape_string($feed) . "'");
1c4421fc
AD
159 }
160
a42c55f0 161 db_query(sprintf("UPDATE ttrss_feeds SET last_update_started = NOW()
1c4421fc 162 WHERE feed_url IN (%s)", implode(',', $feeds_quoted)));
2c08214a
AD
163 }
164
8292d05b 165 $nf = 0;
2d9c5684 166 $bstarted = microtime(true);
8292d05b 167
5cbd1fe8
AD
168 $batch_owners = array();
169
2c08214a 170 // For each feed, we call the feed update function.
1c4421fc
AD
171 foreach ($feeds_to_update as $feed) {
172 if($debug) _debug("Base feed: $feed");
2c08214a 173
a42c55f0 174 //update_rss_feed($line["id"], true);
2c08214a 175
1c4421fc
AD
176 // since we have the data cached, we can deal with other feeds with the same url
177
15c762be 178 $tmp_result = db_query("SELECT DISTINCT ttrss_feeds.id,last_updated,ttrss_feeds.owner_uid
ee0542ce
AD
179 FROM ttrss_feeds, ttrss_users, ttrss_user_prefs WHERE
180 ttrss_user_prefs.owner_uid = ttrss_feeds.owner_uid AND
181 ttrss_users.id = ttrss_user_prefs.owner_uid AND
182 ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' AND
f08426e3 183 ttrss_user_prefs.profile IS NULL AND
9e84bab4
AD
184 feed_url = '".db_escape_string($feed)."'
185 $update_limit_qpart
1c4421fc 186 $login_thresh_qpart
5929a0c1 187 ORDER BY ttrss_feeds.id $query_limit");
1c4421fc
AD
188
189 if (db_num_rows($tmp_result) > 0) {
190 while ($tline = db_fetch_assoc($tmp_result)) {
15c762be 191 if($debug) _debug(" => " . $tline["last_updated"] . ", " . $tline["id"] . " " . $tline["owner_uid"]);
f08426e3 192
5cbd1fe8
AD
193 if (array_search($tline["owner_uid"], $batch_owners) === FALSE)
194 array_push($batch_owners, $tline["owner_uid"]);
195
2d9c5684 196 $fstarted = microtime(true);
7b55001e 197 update_rss_feed($tline["id"], true, false);
4f71d743 198 _debug_suppress(false);
2d9c5684
AD
199
200 _debug(sprintf(" %.4f (sec)", microtime(true) - $fstarted));
201
8292d05b 202 ++$nf;
1c4421fc
AD
203 }
204 }
2c08214a
AD
205 }
206
2d9c5684
AD
207 if ($nf > 0) {
208 _debug(sprintf("Processed %d feeds in %.4f (sec), %.4f (sec/feed avg)", $nf,
209 microtime(true) - $bstarted, (microtime(true) - $bstarted) / $nf));
210 }
211
5cbd1fe8
AD
212 foreach ($batch_owners as $owner_uid) {
213 _debug("Running housekeeping tasks for user $owner_uid...");
214
215 housekeeping_user($owner_uid);
216 }
217
f1611683
AD
218 require_once "digest.php";
219
2c08214a 220 // Send feed digests by email if needed.
a42c55f0 221 send_headlines_digests($debug);
2c08214a 222
8292d05b
AD
223 return $nf;
224
7b55001e 225 }
2c08214a 226
6022776d
AD
227 // this is used when subscribing
228 function set_basic_feed_info($feed) {
229
230 $feed = db_escape_string($feed);
231
9bd422c2 232 $result = db_query("SELECT feed_url,auth_pass,auth_login,auth_pass_encrypted
6022776d
AD
233 FROM ttrss_feeds WHERE id = '$feed'");
234
235 $auth_pass_encrypted = sql_bool_to_bool(db_fetch_result($result,
236 0, "auth_pass_encrypted"));
237
238 $auth_login = db_fetch_result($result, 0, "auth_login");
239 $auth_pass = db_fetch_result($result, 0, "auth_pass");
240
17a8e61d 241 if ($auth_pass_encrypted && function_exists("mcrypt_decrypt")) {
6022776d
AD
242 require_once "crypt.php";
243 $auth_pass = decrypt_string($auth_pass);
244 }
245
246 $fetch_url = db_fetch_result($result, 0, "feed_url");
247
248 $feed_data = fetch_file_contents($fetch_url, false,
249 $auth_login, $auth_pass, false,
3d5d2890 250 FEED_FETCH_TIMEOUT,
6022776d
AD
251 0);
252
253 global $fetch_curl_used;
254
255 if (!$fetch_curl_used) {
256 $tmp = @gzdecode($feed_data);
257
258 if ($tmp) $feed_data = $tmp;
259 }
260
261 $feed_data = trim($feed_data);
262
263 $rss = new FeedParser($feed_data);
264 $rss->init();
265
266 if (!$rss->error()) {
267
268 $result = db_query("SELECT title, site_url FROM ttrss_feeds WHERE id = '$feed'");
269
270 $registered_title = db_fetch_result($result, 0, "title");
271 $orig_site_url = db_fetch_result($result, 0, "site_url");
272
273 $site_url = db_escape_string(mb_substr(rewrite_relative_url($fetch_url, $rss->get_link()), 0, 245));
274 $feed_title = db_escape_string(mb_substr($rss->get_title(), 0, 199));
275
276 if ($feed_title && (!$registered_title || $registered_title == "[Unknown]")) {
277 db_query("UPDATE ttrss_feeds SET
278 title = '$feed_title' WHERE id = '$feed'");
279 }
280
281 if ($site_url && $orig_site_url != $site_url) {
282 db_query("UPDATE ttrss_feeds SET
283 site_url = '$site_url' WHERE id = '$feed'");
284 }
285 }
286 }
287
7b55001e
AD
288 /**
289 * @SuppressWarnings(PHPMD.UnusedFormalParameter)
290 */
291 function update_rss_feed($feed, $no_cache = false) {
2c08214a 292
2c08214a
AD
293 $debug_enabled = defined('DAEMON_EXTENDED_DEBUG') || $_REQUEST['xdebug'];
294
4f71d743 295 _debug_suppress(!$debug_enabled);
68cccafc 296 _debug("start", $debug_enabled);
2c08214a 297
6bb96beb
AD
298 $result = db_query("SELECT title FROM ttrss_feeds
299 WHERE id = '$feed'");
bfe1eb4e
AD
300
301 if (db_num_rows($result) == 0) {
302 _debug("feed $feed NOT FOUND/SKIPPED", $debug_enabled);
303 user_error("Attempt to update unknown/invalid feed $feed", E_USER_WARNING);
304 return false;
305 }
306
6bb96beb
AD
307 $title = db_fetch_result($result, 0, "title");
308
309 // feed was batch-subscribed or something, we need to get basic info
310 // this is not optimal currently as it fetches stuff separately TODO: optimize
311 if ($title == "[Unknown]") {
312 _debug("setting basic feed info for $feed...");
313 set_basic_feed_info($feed);
314 }
315
a42c55f0 316 $result = db_query("SELECT id,update_interval,auth_login,
5ba1ddd4 317 feed_url,auth_pass,cache_images,
5321e775 318 mark_unread_on_update, owner_uid,
90e5f4f1 319 pubsub_state, auth_pass_encrypted,
0edf1d0d 320 feed_language
87764a50 321 FROM ttrss_feeds WHERE id = '$feed'");
2c08214a 322
2c08214a
AD
323 $owner_uid = db_fetch_result($result, 0, "owner_uid");
324 $mark_unread_on_update = sql_bool_to_bool(db_fetch_result($result,
325 0, "mark_unread_on_update"));
2c08214a 326 $pubsub_state = db_fetch_result($result, 0, "pubsub_state");
044cff2d
AD
327 $auth_pass_encrypted = sql_bool_to_bool(db_fetch_result($result,
328 0, "auth_pass_encrypted"));
2c08214a 329
a42c55f0 330 db_query("UPDATE ttrss_feeds SET last_update_started = NOW()
2c08214a
AD
331 WHERE id = '$feed'");
332
333 $auth_login = db_fetch_result($result, 0, "auth_login");
334 $auth_pass = db_fetch_result($result, 0, "auth_pass");
335
17a8e61d 336 if ($auth_pass_encrypted && function_exists("mcrypt_decrypt")) {
044cff2d
AD
337 require_once "crypt.php";
338 $auth_pass = decrypt_string($auth_pass);
339 }
340
2c08214a
AD
341 $cache_images = sql_bool_to_bool(db_fetch_result($result, 0, "cache_images"));
342 $fetch_url = db_fetch_result($result, 0, "feed_url");
df659891
AD
343 $feed_language = db_escape_string(mb_strtolower(db_fetch_result($result, 0, "feed_language")));
344 if (!$feed_language) $feed_language = 'english';
2c08214a 345
a42c55f0 346 $feed = db_escape_string($feed);
2c08214a 347
f074ffe9 348 $date_feed_processed = date('Y-m-d H:i');
2c08214a 349
865a3ed6 350 $cache_filename = CACHE_DIR . "/simplepie/" . sha1($fetch_url) . ".xml";
f074ffe9 351
ee65bef4
AD
352 $pluginhost = new PluginHost();
353 $pluginhost->set_debug($debug_enabled);
354 $user_plugins = get_pref("_ENABLED_PLUGINS", $owner_uid);
355
356 $pluginhost->load(PLUGINS, PluginHost::KIND_ALL);
357 $pluginhost->load($user_plugins, PluginHost::KIND_USER, $owner_uid);
358 $pluginhost->load_data();
359
7b55001e 360 $rss_hash = false;
4f9cbdff 361
7b55001e
AD
362 $force_refetch = isset($_REQUEST["force_refetch"]);
363 $feed_data = "";
687a4f59 364
7b55001e
AD
365 foreach ($pluginhost->get_hooks(PluginHost::HOOK_FETCH_FEED) as $plugin) {
366 $feed_data = $plugin->hook_fetch_feed($feed_data, $fetch_url, $owner_uid, $feed, 0, $auth_login, $auth_pass);
367 }
2c08214a 368
7b55001e
AD
369 // try cache
370 if (!$feed_data &&
371 file_exists($cache_filename) &&
372 is_readable($cache_filename) &&
373 !$auth_login && !$auth_pass &&
374 filemtime($cache_filename) > time() - 30) {
be574731 375
7b55001e 376 _debug("using local cache [$cache_filename].", $debug_enabled);
52637d3b 377
7b55001e 378 @$feed_data = file_get_contents($cache_filename);
f074ffe9 379
7b55001e
AD
380 if ($feed_data) {
381 $rss_hash = sha1($feed_data);
88edaa93 382 }
ee65bef4 383
7b55001e
AD
384 } else {
385 _debug("local cache will not be used for this feed", $debug_enabled);
386 }
312742db 387
7b55001e
AD
388 // fetch feed from source
389 if (!$feed_data) {
390 _debug("fetching [$fetch_url]...", $debug_enabled);
312742db 391
7b55001e
AD
392 if (ini_get("open_basedir") && function_exists("curl_init")) {
393 _debug("not using CURL due to open_basedir restrictions");
394 }
3f6f0857 395
7b55001e
AD
396 $feed_data = fetch_file_contents($fetch_url, false,
397 $auth_login, $auth_pass, false,
398 $no_cache ? FEED_FETCH_NO_CACHE_TIMEOUT : FEED_FETCH_TIMEOUT,
399 0);
3f6f0857 400
7b55001e 401 global $fetch_curl_used;
3f6f0857 402
7b55001e
AD
403 if (!$fetch_curl_used) {
404 $tmp = @gzdecode($feed_data);
1367bc3f 405
7b55001e
AD
406 if ($tmp) $feed_data = $tmp;
407 }
017401dd 408
7b55001e 409 $feed_data = trim($feed_data);
fd687300 410
7b55001e 411 _debug("fetch done.", $debug_enabled);
95beaa14 412
7b55001e
AD
413 // cache vanilla feed data for re-use
414 if ($feed_data && !$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) {
415 $new_rss_hash = sha1($feed_data);
416
417 if ($new_rss_hash != $rss_hash) {
418 _debug("saving $cache_filename", $debug_enabled);
419 @file_put_contents($cache_filename, $feed_data);
95beaa14 420 }
4f9cbdff 421 }
7b55001e 422 }
017401dd 423
7b55001e
AD
424 if (!$feed_data) {
425 global $fetch_last_error;
426 global $fetch_last_error_code;
f074ffe9 427
7b55001e 428 _debug("unable to fetch: $fetch_last_error [$fetch_last_error_code]", $debug_enabled);
f074ffe9 429
7b55001e 430 $error_escaped = '';
7a01dc77 431
7b55001e
AD
432 // If-Modified-Since
433 if ($fetch_last_error_code != 304) {
434 $error_escaped = db_escape_string($fetch_last_error);
435 } else {
436 _debug("source claims data not modified, nothing to do.", $debug_enabled);
437 }
4f9cbdff 438
7b55001e
AD
439 db_query(
440 "UPDATE ttrss_feeds SET last_error = '$error_escaped',
441 last_updated = NOW() WHERE id = '$feed'");
4f9cbdff 442
7b55001e 443 return;
f074ffe9
AD
444 }
445
1ffe3391 446 foreach ($pluginhost->get_hooks(PluginHost::HOOK_FEED_FETCHED) as $plugin) {
6791af0c 447 $feed_data = $plugin->hook_feed_fetched($feed_data, $fetch_url, $owner_uid, $feed);
017401dd
AD
448 }
449
5ef84097
AD
450 // set last update to now so if anything *simplepie* crashes later we won't be
451 // continuously failing on the same feed
5de40104 452 //db_query("UPDATE ttrss_feeds SET last_updated = NOW() WHERE id = '$feed'");
5ef84097 453
4f9cbdff 454 if (!$rss) {
cd07592c
AD
455 $rss = new FeedParser($feed_data);
456 $rss->init();
5de51df7
AD
457 }
458
2c08214a
AD
459// print_r($rss);
460
a42c55f0 461 $feed = db_escape_string($feed);
2c08214a 462
19b3992b 463 if (!$rss->error()) {
2c08214a 464
d2a421e3 465 // We use local pluginhost here because we need to load different per-user feed plugins
1ffe3391 466 $pluginhost->run_hooks(PluginHost::HOOK_FEED_PARSED, "hook_feed_parsed", $rss);
4412b877 467
df659891 468 _debug("language: $feed_language", $debug_enabled);
68cccafc 469 _debug("processing feed data...", $debug_enabled);
2c08214a 470
a42c55f0 471// db_query("BEGIN");
2c08214a 472
382268c6
AD
473 if (DB_TYPE == "pgsql") {
474 $favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'";
475 } else {
476 $favicon_interval_qpart = "favicon_last_checked < DATE_SUB(NOW(), INTERVAL 12 HOUR)";
477 }
478
fe4535e6 479 $result = db_query("SELECT owner_uid,favicon_avg_color,
382268c6
AD
480 (favicon_last_checked IS NULL OR $favicon_interval_qpart) AS
481 favicon_needs_check
2c08214a
AD
482 FROM ttrss_feeds WHERE id = '$feed'");
483
382268c6
AD
484 $favicon_needs_check = sql_bool_to_bool(db_fetch_result($result, 0,
485 "favicon_needs_check"));
36490f11 486 $favicon_avg_color = db_fetch_result($result, 0, "favicon_avg_color");
2c08214a
AD
487
488 $owner_uid = db_fetch_result($result, 0, "owner_uid");
489
a42c55f0 490 $site_url = db_escape_string(mb_substr(rewrite_relative_url($fetch_url, $rss->get_link()), 0, 245));
2c08214a 491
cd07592c
AD
492 _debug("site_url: $site_url", $debug_enabled);
493 _debug("feed_title: " . $rss->get_title(), $debug_enabled);
494
687a4f59 495 if ($favicon_needs_check || $force_refetch) {
36490f11
AD
496
497 /* terrible hack: if we crash on floicon shit here, we won't check
560cbd8c 498 * the icon avgcolor again (unless the icon got updated) */
36490f11 499
560cbd8c
AD
500 $favicon_file = ICONS_DIR . "/$feed.ico";
501 $favicon_modified = @filemtime($favicon_file);
502
68cccafc 503 _debug("checking favicon...", $debug_enabled);
687a4f59 504
6f7798b6 505 check_feed_favicon($site_url, $feed);
560cbd8c
AD
506 $favicon_modified_new = @filemtime($favicon_file);
507
508 if ($favicon_modified_new > $favicon_modified)
509 $favicon_avg_color = '';
687a4f59 510
6ee0d4b0 511 if (file_exists($favicon_file) && function_exists("imagecreatefromstring") && $favicon_avg_color == '') {
bc7a144d 512 require_once "colors.php";
687a4f59 513
aafd55ba
AD
514 db_query("UPDATE ttrss_feeds SET favicon_avg_color = 'fail' WHERE
515 id = '$feed'");
516
6322ac79 517 $favicon_color = db_escape_string(
bc7a144d 518 calculate_avg_color($favicon_file));
63c323f7 519
bc7a144d 520 $favicon_colorstring = ",favicon_avg_color = '".$favicon_color."'";
36490f11 521 } else if ($favicon_avg_color == 'fail') {
84ceb6bd 522 _debug("floicon failed on this file, not trying to recalculate avg color", $debug_enabled);
6ac722d5 523 }
687a4f59 524
a42c55f0 525 db_query("UPDATE ttrss_feeds SET favicon_last_checked = NOW()
bc7a144d 526 $favicon_colorstring
f2798eb6
AD
527 WHERE id = '$feed'");
528 }
2c08214a 529
68cccafc 530 _debug("loading filters & labels...", $debug_enabled);
2c08214a 531
a42c55f0 532 $filters = load_filters($feed, $owner_uid);
2c08214a 533
68cccafc 534 _debug("" . count($filters) . " filters loaded.", $debug_enabled);
2c08214a 535
19b3992b 536 $items = $rss->get_items();
2c08214a 537
19b3992b 538 if (!is_array($items)) {
68cccafc 539 _debug("no articles found.", $debug_enabled);
2c08214a 540
a42c55f0 541 db_query("UPDATE ttrss_feeds
2c08214a
AD
542 SET last_updated = NOW(), last_error = '' WHERE id = '$feed'");
543
544 return; // no articles
545 }
546
547 if ($pubsub_state != 2 && PUBSUBHUBBUB_ENABLED) {
548
68cccafc 549 _debug("checking for PUSH hub...", $debug_enabled);
2c08214a
AD
550
551 $feed_hub_url = false;
2c08214a 552
19b3992b 553 $links = $rss->get_links('hub');
2c08214a 554
19b3992b
AD
555 if ($links && is_array($links)) {
556 foreach ($links as $l) {
557 $feed_hub_url = $l;
558 break;
2c08214a
AD
559 }
560 }
561
68cccafc 562 _debug("feed hub url: $feed_hub_url", $debug_enabled);
2c08214a 563
d8179cb9
AD
564 $feed_self_url = $fetch_url;
565
566 $links = $rss->get_links('self');
567
568 if ($links && is_array($links)) {
569 foreach ($links as $l) {
570 $feed_self_url = $l;
571 break;
572 }
573 }
574
575 _debug("feed self url = $feed_self_url");
576
577 if ($feed_hub_url && $feed_self_url && function_exists('curl_init') &&
2c08214a
AD
578 !ini_get("open_basedir")) {
579
becd215a 580 require_once 'lib/pubsubhubbub/Subscriber.php';
2c08214a
AD
581
582 $callback_url = get_self_url_prefix() .
583 "/public.php?op=pubsub&id=$feed";
584
cb3f8773 585 $s = new Pubsubhubbub\Subscriber\Subscriber($feed_hub_url, $callback_url);
2c08214a 586
d8179cb9 587 $rc = $s->subscribe($feed_self_url);
2c08214a 588
5ba1ddd4 589 _debug("feed hub url found, subscribe request sent. [rc=$rc]", $debug_enabled);
2c08214a 590
a42c55f0 591 db_query("UPDATE ttrss_feeds SET pubsub_state = 1
2c08214a
AD
592 WHERE id = '$feed'");
593 }
594 }
595
68cccafc 596 _debug("processing articles...", $debug_enabled);
2c08214a 597
6c9f3d4a
AD
598 $tstart = time();
599
19b3992b 600 foreach ($items as $item) {
5d56d100 601 if ($_REQUEST['xdebug'] == 3) {
2c08214a
AD
602 print_r($item);
603 }
604
6c9f3d4a
AD
605 if (ini_get("max_execution_time") > 0 && time() - $tstart >= ini_get("max_execution_time") * 0.7) {
606 _debug("looks like there's too many articles to process at once, breaking out", $debug_enabled);
607 break;
608 }
609
19b3992b
AD
610 $entry_guid = $item->get_id();
611 if (!$entry_guid) $entry_guid = $item->get_link();
612 if (!$entry_guid) $entry_guid = make_guid_from_title($item->get_title());
2c08214a
AD
613 if (!$entry_guid) continue;
614
3a4c8973
AD
615 $entry_guid = "$owner_uid,$entry_guid";
616
a42c55f0 617 $entry_guid_hashed = db_escape_string('SHA1:' . sha1($entry_guid));
5e3d5480 618
68cccafc 619 _debug("guid $entry_guid / $entry_guid_hashed", $debug_enabled);
5e3d5480 620
2c08214a
AD
621 $entry_timestamp = "";
622
04d2f9c8
AD
623 $entry_timestamp = $item->get_date();
624
625 _debug("orig date: " . $item->get_date(), $debug_enabled);
2c08214a 626
30123fe6 627 if ($entry_timestamp == -1 || !$entry_timestamp || $entry_timestamp > time()) {
2c08214a 628 $entry_timestamp = time();
2c08214a
AD
629 }
630
631 $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp);
632
68cccafc 633 _debug("date $entry_timestamp [$entry_timestamp_fmt]", $debug_enabled);
2c08214a 634
99429e57
AD
635// $entry_title = html_entity_decode($item->get_title(), ENT_COMPAT, 'UTF-8');
636// $entry_title = decode_numeric_entities($entry_title);
637 $entry_title = $item->get_title();
1b35d30c 638
5d56d100 639 $entry_link = rewrite_relative_url($site_url, $item->get_link());
2c08214a 640
68cccafc
AD
641 _debug("title $entry_title", $debug_enabled);
642 _debug("link $entry_link", $debug_enabled);
2c08214a
AD
643
644 if (!$entry_title) $entry_title = date("Y-m-d H:i:s", $entry_timestamp);;
645
19b3992b
AD
646 $entry_content = $item->get_content();
647 if (!$entry_content) $entry_content = $item->get_description();
2c08214a
AD
648
649 if ($_REQUEST["xdebug"] == 2) {
9ec10352 650 print "content: ";
0bc503ff 651 print htmlspecialchars($entry_content);
3c696512 652 print "\n";
2c08214a
AD
653 }
654
12ff230b
AD
655 $entry_comments = db_escape_string(mb_substr($item->get_comments_url(), 0, 245));
656 $num_comments = (int) $item->get_comments_count();
2c08214a 657
12ff230b 658 $entry_author = $item->get_author(); // escaped later
a42c55f0 659 $entry_guid = db_escape_string(mb_substr($entry_guid, 0, 245));
2c08214a 660
68cccafc
AD
661 _debug("author $entry_author", $debug_enabled);
662 _debug("num_comments: $num_comments", $debug_enabled);
ee78f81c 663 _debug("looking for tags...", $debug_enabled);
2c08214a
AD
664
665 // parse <category> entries into tags
666
667 $additional_tags = array();
668
19b3992b 669 $additional_tags_src = $item->get_categories();
2c08214a 670
19b3992b
AD
671 if (is_array($additional_tags_src)) {
672 foreach ($additional_tags_src as $tobj) {
cd07592c 673 array_push($additional_tags, $tobj);
2c08214a 674 }
19b3992b 675 }
2c08214a 676
fa6fbd36 677 $entry_tags = array_unique($additional_tags);
2c08214a
AD
678
679 for ($i = 0; $i < count($entry_tags); $i++)
680 $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8');
681
ee78f81c
AD
682 _debug("tags found: " . join(",", $entry_tags), $debug_enabled);
683
68cccafc 684 _debug("done collecting data.", $debug_enabled);
2c08214a 685
3318d324 686 $result = db_query("SELECT id, content_hash, lang FROM ttrss_entries
b1840673 687 WHERE guid = '".db_escape_string($entry_guid)."' OR guid = '$entry_guid_hashed'");
b30abdad
AD
688
689 if (db_num_rows($result) != 0) {
b1840673
AD
690 $base_entry_id = db_fetch_result($result, 0, "id");
691 $entry_stored_hash = db_fetch_result($result, 0, "content_hash");
a29fe121 692 $article_labels = get_article_labels($base_entry_id, $owner_uid);
3318d324 693 $entry_language = db_fetch_result($result, 0, "lang");
a8ac7661 694
040aa540 695 $existing_tags = get_article_tags($base_entry_id, $owner_uid);
a8ac7661
AD
696 $entry_tags = array_unique(array_merge($entry_tags, $existing_tags));
697
b30abdad 698 } else {
b1840673
AD
699 $base_entry_id = false;
700 $entry_stored_hash = "";
a29fe121 701 $article_labels = array();
3318d324 702 $entry_language = "";
b30abdad
AD
703 }
704
455b1401 705 $article = array("owner_uid" => $owner_uid, // read only
b30abdad 706 "guid" => $entry_guid, // read only
59e83455 707 "guid_hashed" => $entry_guid_hashed, // read only
19b3992b
AD
708 "title" => $entry_title,
709 "content" => $entry_content,
710 "link" => $entry_link,
a29fe121 711 "labels" => $article_labels, // current limitation: can add labels to article, can't remove them
19b3992b 712 "tags" => $entry_tags,
e02555c1 713 "author" => $entry_author,
c9299c28 714 "force_catchup" => false, // ugly hack for the time being
6de3a1be 715 "score_modifier" => 0, // no previous value, plugin should recalculate score modifier based on content if needed
3318d324 716 "language" => $entry_language,
f73e03e0
AD
717 "feed" => array("id" => $feed,
718 "fetch_url" => $fetch_url,
babfadbf
J
719 "site_url" => $site_url,
720 "cache_images" => $cache_images)
f73e03e0 721 );
cc85704f 722
b1840673 723 $entry_plugin_data = "";
eb16bd9f 724 $entry_current_hash = calculate_article_hash($article, $pluginhost);
b1840673
AD
725
726 _debug("article hash: $entry_current_hash [stored=$entry_stored_hash]", $debug_enabled);
727
522e8b35 728 if ($entry_current_hash == $entry_stored_hash && !isset($_REQUEST["force_rehash"])) {
b1840673
AD
729 _debug("stored article seems up to date [IID: $base_entry_id], updating timestamp only", $debug_enabled);
730
731 // we keep encountering the entry in feeds, so we need to
732 // update date_updated column so that we don't get horrible
733 // dupes when the entry gets purged and reinserted again e.g.
734 // in the case of SLOW SLOW OMG SLOW updating feeds
735
736 $base_entry_id = db_fetch_result($result, 0, "id");
737
738 db_query("UPDATE ttrss_entries SET date_updated = NOW()
739 WHERE id = '$base_entry_id'");
740
5bdcb8fd 741 continue;
b1840673
AD
742 }
743
744 _debug("hash differs, applying plugin filters:", $debug_enabled);
745
1ffe3391 746 foreach ($pluginhost->get_hooks(PluginHost::HOOK_ARTICLE_FILTER) as $plugin) {
b1840673
AD
747 _debug("... " . get_class($plugin), $debug_enabled);
748
749 $start = microtime(true);
19b3992b 750 $article = $plugin->hook_article_filter($article);
0084f0d1 751
b1840673
AD
752 _debug("=== " . sprintf("%.4f (sec)", microtime(true) - $start), $debug_enabled);
753
754 $entry_plugin_data .= mb_strtolower(get_class($plugin)) . ",";
cc85704f
AD
755 }
756
0bc503ff
AD
757 if ($_REQUEST["xdebug"] == 2) {
758 print "processed content: ";
759 print htmlspecialchars($article["content"]);
760 print "\n";
761 }
762
b1840673
AD
763 $entry_plugin_data = db_escape_string($entry_plugin_data);
764
765 _debug("plugin data: $entry_plugin_data", $debug_enabled);
766
35c12dc4
AD
767 // Workaround: 4-byte unicode requires utf8mb4 in MySQL. See https://tt-rss.org/forum/viewtopic.php?f=1&t=3377&p=20077#p20077
768 if (DB_TYPE == "mysql") {
769 foreach ($article as $k => $v) {
35c37354
AD
770
771 // i guess we'll have to take the risk of 4byte unicode labels & tags here
dae16f72 772 if (is_string($article[$k])) {
35c37354
AD
773 $article[$k] = preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $v);
774 }
35c12dc4
AD
775 }
776 }
777
b8774453
AD
778 /* Collect article tags here so we could filter by them: */
779
557d86fe
AD
780 $matched_rules = array();
781
b8774453 782 $article_filters = get_article_filters($filters, $article["title"],
7b55001e 783 $article["content"], $article["link"], $article["author"],
557d86fe 784 $article["tags"], $matched_rules);
b8774453
AD
785
786 if ($debug_enabled) {
557d86fe
AD
787 _debug("matched filter rules: ", $debug_enabled);
788
789 if (count($matched_rules) != 0) {
790 print_r($matched_rules);
791 }
792
793 _debug("filter actions: ", $debug_enabled);
794
b8774453
AD
795 if (count($article_filters) != 0) {
796 print_r($article_filters);
797 }
798 }
799
800 $plugin_filter_names = find_article_filters($article_filters, "plugin");
801 $plugin_filter_actions = $pluginhost->get_filter_actions();
802
803 if (count($plugin_filter_names) > 0) {
804 _debug("applying plugin filter actions...", $debug_enabled);
805
806 foreach ($plugin_filter_names as $pfn) {
807 list($pfclass,$pfaction) = explode(":", $pfn["param"]);
808
809 if (isset($plugin_filter_actions[$pfclass])) {
810 $plugin = $pluginhost->get_plugin($pfclass);
811
812 _debug("... $pfclass: $pfaction", $debug_enabled);
813
814 if ($plugin) {
815 $start = microtime(true);
816 $article = $plugin->hook_article_filter_action($article, $pfaction);
817
818 _debug("=== " . sprintf("%.4f (sec)", microtime(true) - $start), $debug_enabled);
819 } else {
820 _debug("??? $pfclass: plugin object not found.");
821 }
822 } else {
823 _debug("??? $pfclass: filter plugin not registered.");
824 }
825 }
826 }
827
19b3992b 828 $entry_tags = $article["tags"];
a42c55f0
AD
829 $entry_guid = db_escape_string($entry_guid);
830 $entry_title = db_escape_string($article["title"]);
12ff230b 831 $entry_author = db_escape_string(mb_substr($article["author"], 0, 245));
a42c55f0 832 $entry_link = db_escape_string($article["link"]);
f935d98e 833 $entry_content = $article["content"]; // escaped below
c9299c28 834 $entry_force_catchup = $article["force_catchup"];
a29fe121 835 $article_labels = $article["labels"];
6de3a1be 836 $entry_score_modifier = (int) $article["score_modifier"];
3318d324 837 $entry_language = db_escape_string($article["language"]);
a29fe121
AD
838
839 if ($debug_enabled) {
840 _debug("article labels:", $debug_enabled);
557d86fe
AD
841
842 if (count($article_labels) != 0) {
843 print_r($article_labels);
844 }
a29fe121 845 }
c9299c28
AD
846
847 _debug("force catchup: $entry_force_catchup");
f935d98e 848
0a3fd79b 849 if ($cache_images && is_writable(CACHE_DIR . '/images'))
41bead9b 850 cache_media($entry_content, $site_url, $debug_enabled);
0a3fd79b 851
a42c55f0 852 $entry_content = db_escape_string($entry_content, false);
1f45c857 853
07877caf 854 //db_query("BEGIN");
2c08214a 855
a42c55f0 856 $result = db_query("SELECT id FROM ttrss_entries
5e3d5480 857 WHERE (guid = '$entry_guid' OR guid = '$entry_guid_hashed')");
9e222305 858
2c08214a
AD
859 if (db_num_rows($result) == 0) {
860
68cccafc 861 _debug("base guid [$entry_guid] not found", $debug_enabled);
2c08214a
AD
862
863 // base post entry does not exist, create it
864
6322ac79 865 $result = db_query(
2c08214a
AD
866 "INSERT INTO ttrss_entries
867 (title,
868 guid,
869 link,
870 updated,
871 content,
872 content_hash,
873 no_orig_date,
874 date_updated,
875 date_entered,
876 comments,
877 num_comments,
b30abdad 878 plugin_data,
6b461797 879 lang,
2c08214a
AD
880 author)
881 VALUES
882 ('$entry_title',
5e3d5480 883 '$entry_guid_hashed',
2c08214a
AD
884 '$entry_link',
885 '$entry_timestamp_fmt',
886 '$entry_content',
b1840673 887 '$entry_current_hash',
5ba1ddd4 888 false,
2c08214a 889 NOW(),
be574731 890 '$date_feed_processed',
2c08214a
AD
891 '$entry_comments',
892 '$num_comments',
b30abdad 893 '$entry_plugin_data',
6b461797 894 '$entry_language',
2c08214a 895 '$entry_author')");
e8291805 896
2c08214a 897 } else {
2c08214a 898 $base_entry_id = db_fetch_result($result, 0, "id");
2c08214a
AD
899 }
900
901 // now it should exist, if not - bad luck then
902
b1840673 903 $result = db_query("SELECT id FROM ttrss_entries
5e3d5480 904 WHERE guid = '$entry_guid' OR guid = '$entry_guid_hashed'");
2c08214a
AD
905
906 $entry_ref_id = 0;
907 $entry_int_id = 0;
908
909 if (db_num_rows($result) == 1) {
910
68cccafc 911 _debug("base guid found, checking for user record", $debug_enabled);
2c08214a 912
2c08214a
AD
913 $ref_id = db_fetch_result($result, 0, "id");
914 $entry_ref_id = $ref_id;
915
5e3d5480
AD
916 /* $stored_guid = db_fetch_result($result, 0, "guid");
917 if ($stored_guid != $entry_guid_hashed) {
68cccafc 918 if ($debug_enabled) _debug("upgrading compat guid to hashed one", $debug_enabled);
5e3d5480 919
a42c55f0 920 db_query("UPDATE ttrss_entries SET guid = '$entry_guid_hashed' WHERE
5e3d5480
AD
921 id = '$ref_id'");
922 } */
923
2c08214a 924 if (find_article_filter($article_filters, "filter")) {
07877caf 925 //db_query("COMMIT"); // close transaction in progress
2c08214a
AD
926 continue;
927 }
928
6de3a1be 929 $score = calculate_article_score($article_filters) + $entry_score_modifier;
2c08214a 930
6de3a1be 931 _debug("initial score: $score [including plugin modifier: $entry_score_modifier]", $debug_enabled);
2c08214a 932
4f186b1f
AD
933 // check for user post link to main table
934
2c08214a 935 $query = "SELECT ref_id, int_id FROM ttrss_user_entries WHERE
5bdcb8fd 936 ref_id = '$ref_id' AND owner_uid = '$owner_uid'";
2c08214a
AD
937
938// if ($_REQUEST["xdebug"]) print "$query\n";
939
a42c55f0 940 $result = db_query($query);
2c08214a
AD
941
942 // okay it doesn't exist - create user entry
943 if (db_num_rows($result) == 0) {
944
68cccafc 945 _debug("user record not found, creating...", $debug_enabled);
2c08214a 946
c9299c28 947 if ($score >= -500 && !find_article_filter($article_filters, 'catchup') && !$entry_force_catchup) {
2c08214a
AD
948 $unread = 'true';
949 $last_read_qpart = 'NULL';
950 } else {
951 $unread = 'false';
952 $last_read_qpart = 'NOW()';
953 }
954
955 if (find_article_filter($article_filters, 'mark') || $score > 1000) {
956 $marked = 'true';
957 } else {
958 $marked = 'false';
959 }
960
961 if (find_article_filter($article_filters, 'publish')) {
962 $published = 'true';
963 } else {
964 $published = 'false';
965 }
966
7873d588
AD
967 $last_marked = ($marked == 'true') ? 'NOW()' : 'NULL';
968 $last_published = ($published == 'true') ? 'NOW()' : 'NULL';
969
6322ac79 970 $result = db_query(
2c08214a
AD
971 "INSERT INTO ttrss_user_entries
972 (ref_id, owner_uid, feed_id, unread, last_read, marked,
7873d588
AD
973 published, score, tag_cache, label_cache, uuid,
974 last_marked, last_published)
2c08214a 975 VALUES ('$ref_id', '$owner_uid', '$feed', $unread,
7873d588
AD
976 $last_read_qpart, $marked, $published, '$score', '', '',
977 '', $last_marked, $last_published)");
2c08214a
AD
978
979 if (PUBSUBHUBBUB_HUB && $published == 'true') {
980 $rss_link = get_self_url_prefix() .
981 "/public.php?op=rss&id=-2&key=" .
a42c55f0 982 get_feed_access_key(-2, false, $owner_uid);
2c08214a 983
cb3f8773 984 $p = new pubsubhubbub\publisher\Publisher(PUBSUBHUBBUB_HUB);
2c08214a 985
5ba1ddd4 986 /* $pubsub_result = */ $p->publish_update($rss_link);
2c08214a
AD
987 }
988
6322ac79 989 $result = db_query(
2c08214a
AD
990 "SELECT int_id FROM ttrss_user_entries WHERE
991 ref_id = '$ref_id' AND owner_uid = '$owner_uid' AND
992 feed_id = '$feed' LIMIT 1");
993
994 if (db_num_rows($result) == 1) {
995 $entry_int_id = db_fetch_result($result, 0, "int_id");
996 }
997 } else {
68cccafc 998 _debug("user record FOUND", $debug_enabled);
2c08214a
AD
999
1000 $entry_ref_id = db_fetch_result($result, 0, "ref_id");
1001 $entry_int_id = db_fetch_result($result, 0, "int_id");
1002 }
1003
68cccafc 1004 _debug("RID: $entry_ref_id, IID: $entry_int_id", $debug_enabled);
2c08214a 1005
e854442e 1006 if (DB_TYPE == "pgsql") {
978989bb 1007 $tsvector_combined = db_escape_string(mb_substr($entry_title . ' ' . strip_tags(str_replace('<', ' <', $entry_content)),
e854442e
AD
1008 0, 1000000));
1009
df659891 1010 $tsvector_qpart = "tsvector_combined = to_tsvector('$feed_language', '$tsvector_combined'),";
e854442e
AD
1011
1012 } else {
1013 $tsvector_qpart = "";
1014 }
1015
b1840673
AD
1016 db_query("UPDATE ttrss_entries
1017 SET title = '$entry_title',
1018 content = '$entry_content',
1019 content_hash = '$entry_current_hash',
1020 updated = '$entry_timestamp_fmt',
e854442e 1021 $tsvector_qpart
b1840673
AD
1022 num_comments = '$num_comments',
1023 plugin_data = '$entry_plugin_data',
1024 author = '$entry_author',
1025 lang = '$entry_language'
1026 WHERE id = '$ref_id'");
1027
59e83455
AD
1028 // update aux data
1029 db_query("UPDATE ttrss_user_entries
1030 SET score = '$score' WHERE ref_id = '$ref_id'");
1031
b1840673 1032 if ($mark_unread_on_update) {
24e6ff5d
AD
1033 _debug("article updated, marking unread as requested.", $debug_enabled);
1034
b1840673
AD
1035 db_query("UPDATE ttrss_user_entries
1036 SET last_read = null, unread = true WHERE ref_id = '$ref_id'");
2c08214a
AD
1037 }
1038 }
1039
07877caf 1040 //db_query("COMMIT");
2c08214a 1041
a29fe121
AD
1042 _debug("assigning labels [other]...", $debug_enabled);
1043
1044 foreach ($article_labels as $label) {
1045 label_add_article($entry_ref_id, $label[1], $owner_uid);
1046 }
1047
1048 _debug("assigning labels [filters]...", $debug_enabled);
2c08214a 1049
a42c55f0 1050 assign_article_to_label_filters($entry_ref_id, $article_filters,
b24504b1 1051 $owner_uid, $article_labels);
2c08214a 1052
68cccafc 1053 _debug("looking for enclosures...", $debug_enabled);
2c08214a
AD
1054
1055 // enclosures
1056
1057 $enclosures = array();
1058
19b3992b 1059 $encs = $item->get_enclosures();
2c08214a 1060
19b3992b
AD
1061 if (is_array($encs)) {
1062 foreach ($encs as $e) {
1063 $e_item = array(
86e53429
AD
1064 rewrite_relative_url($site_url, $e->link),
1065 $e->type, $e->length, $e->title, $e->width, $e->height);
2c08214a 1066 array_push($enclosures, $e_item);
2c08214a
AD
1067 }
1068 }
1069
388d4dfa
AD
1070 if ($cache_images && is_writable(CACHE_DIR . '/images'))
1071 cache_enclosures($enclosures, $site_url, $debug_enabled);
1072
2c08214a 1073 if ($debug_enabled) {
68cccafc 1074 _debug("article enclosures:", $debug_enabled);
2c08214a
AD
1075 print_r($enclosures);
1076 }
1077
dab229cd 1078 //db_query("BEGIN");
2c08214a 1079
5c54e683
AD
1080// debugging
1081// db_query("DELETE FROM ttrss_enclosures WHERE post_id = '$entry_ref_id'");
1082
2c08214a 1083 foreach ($enclosures as $enc) {
a42c55f0
AD
1084 $enc_url = db_escape_string($enc[0]);
1085 $enc_type = db_escape_string($enc[1]);
1086 $enc_dur = db_escape_string($enc[2]);
5c54e683 1087 $enc_title = db_escape_string($enc[3]);
523bd90b
FE
1088 $enc_width = intval($enc[4]);
1089 $enc_height = intval($enc[5]);
2c08214a 1090
a42c55f0 1091 $result = db_query("SELECT id FROM ttrss_enclosures
2c08214a
AD
1092 WHERE content_url = '$enc_url' AND post_id = '$entry_ref_id'");
1093
1094 if (db_num_rows($result) == 0) {
a42c55f0 1095 db_query("INSERT INTO ttrss_enclosures
523bd90b
FE
1096 (content_url, content_type, title, duration, post_id, width, height) VALUES
1097 ('$enc_url', '$enc_type', '$enc_title', '$enc_dur', '$entry_ref_id', $enc_width, $enc_height)");
2c08214a
AD
1098 }
1099 }
1100
dab229cd 1101 //db_query("COMMIT");
2c08214a
AD
1102
1103 // check for manual tags (we have to do it here since they're loaded from filters)
1104
1105 foreach ($article_filters as $f) {
6aff7845 1106 if ($f["type"] == "tag") {
2c08214a 1107
6aff7845 1108 $manual_tags = trim_array(explode(",", $f["param"]));
2c08214a
AD
1109
1110 foreach ($manual_tags as $tag) {
1111 if (tag_is_valid($tag)) {
1112 array_push($entry_tags, $tag);
1113 }
1114 }
1115 }
1116 }
1117
1118 // Skip boring tags
1119
6322ac79 1120 $boring_tags = trim_array(explode(",", mb_strtolower(get_pref(
2c08214a
AD
1121 'BLACKLISTED_TAGS', $owner_uid, ''), 'utf-8')));
1122
1123 $filtered_tags = array();
1124 $tags_to_cache = array();
1125
1126 if ($entry_tags && is_array($entry_tags)) {
1127 foreach ($entry_tags as $tag) {
1128 if (array_search($tag, $boring_tags) === false) {
1129 array_push($filtered_tags, $tag);
1130 }
1131 }
1132 }
1133
1134 $filtered_tags = array_unique($filtered_tags);
1135
1136 if ($debug_enabled) {
68cccafc 1137 _debug("filtered article tags:", $debug_enabled);
2c08214a
AD
1138 print_r($filtered_tags);
1139 }
1140
1141 // Save article tags in the database
1142
1143 if (count($filtered_tags) > 0) {
1144
dab229cd 1145 //db_query("BEGIN");
2c08214a
AD
1146
1147 foreach ($filtered_tags as $tag) {
1148
1149 $tag = sanitize_tag($tag);
a42c55f0 1150 $tag = db_escape_string($tag);
2c08214a
AD
1151
1152 if (!tag_is_valid($tag)) continue;
1153
a42c55f0 1154 $result = db_query("SELECT id FROM ttrss_tags
2c08214a
AD
1155 WHERE tag_name = '$tag' AND post_int_id = '$entry_int_id' AND
1156 owner_uid = '$owner_uid' LIMIT 1");
1157
1158 if ($result && db_num_rows($result) == 0) {
1159
a42c55f0 1160 db_query("INSERT INTO ttrss_tags
2c08214a
AD
1161 (owner_uid,tag_name,post_int_id)
1162 VALUES ('$owner_uid','$tag', '$entry_int_id')");
1163 }
1164
1165 array_push($tags_to_cache, $tag);
1166 }
1167
1168 /* update the cache */
1169
1170 $tags_to_cache = array_unique($tags_to_cache);
1171
a42c55f0 1172 $tags_str = db_escape_string(join(",", $tags_to_cache));
2c08214a 1173
a42c55f0 1174 db_query("UPDATE ttrss_user_entries
2c08214a
AD
1175 SET tag_cache = '$tags_str' WHERE ref_id = '$entry_ref_id'
1176 AND owner_uid = $owner_uid");
1177
dab229cd 1178 //db_query("COMMIT");
2c08214a
AD
1179 }
1180
68cccafc 1181 _debug("article processed", $debug_enabled);
2c08214a
AD
1182 }
1183
68cccafc 1184 _debug("purging feed...", $debug_enabled);
2c08214a 1185
a42c55f0 1186 purge_feed($feed, 0, $debug_enabled);
2c08214a 1187
a42c55f0 1188 db_query("UPDATE ttrss_feeds
f2798eb6 1189 SET last_updated = NOW(), last_error = '' WHERE id = '$feed'");
2c08214a 1190
a42c55f0 1191// db_query("COMMIT");
2c08214a
AD
1192
1193 } else {
1194
a42c55f0 1195 $error_msg = db_escape_string(mb_substr($rss->error(), 0, 245));
2c08214a 1196
4ad04ee2
AD
1197 _debug("fetch error: $error_msg", $debug_enabled);
1198
1199 if (count($rss->errors()) > 1) {
1200 foreach ($rss->errors() as $error) {
1201 _debug("+ $error");
1202 }
1203 }
2c08214a 1204
6322ac79 1205 db_query(
2c08214a 1206 "UPDATE ttrss_feeds SET last_error = '$error_msg',
88edaa93 1207 last_updated = NOW() WHERE id = '$feed'");
2c08214a 1208
88edaa93 1209 unset($rss);
7b55001e 1210 return;
88edaa93 1211 }
2c08214a 1212
68cccafc 1213 _debug("done", $debug_enabled);
88edaa93 1214
7b55001e 1215 return true;
2c08214a
AD
1216 }
1217
388d4dfa
AD
1218 function cache_enclosures($enclosures, $site_url, $debug) {
1219 foreach ($enclosures as $enc) {
1220
1221 if (preg_match("/(image|audio|video)/", $enc[1])) {
1222
1223 $src = rewrite_relative_url($site_url, $enc[0]);
1224
1225 $local_filename = CACHE_DIR . "/images/" . sha1($src);
1226
1227 if ($debug) _debug("cache_enclosures: downloading: $src to $local_filename");
1228
1229 if (!file_exists($local_filename)) {
1230 $file_content = fetch_file_contents($src);
1231
1232 if ($file_content && strlen($file_content) > _MIN_CACHE_FILE_SIZE) {
1233 file_put_contents($local_filename, $file_content);
1234 }
1235 } else {
1236 touch($local_filename);
1237 }
1238 }
1239 }
1240 }
1241
41bead9b 1242 function cache_media($html, $site_url, $debug) {
3c696512
AD
1243 libxml_use_internal_errors(true);
1244
1245 $charset_hack = '<head>
1246 <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
1247 </head>';
1248
1249 $doc = new DOMDocument();
1250 $doc->loadHTML($charset_hack . $html);
1251 $xpath = new DOMXPath($doc);
1252
388d4dfa 1253 $entries = $xpath->query('(//img[@src])|(//video/source[@src])|(//audio/source[@src])');
3c696512
AD
1254
1255 foreach ($entries as $entry) {
5edd605a 1256 if ($entry->hasAttribute('src') && strpos($entry->getAttribute('src'), "data:") !== 0) {
3c696512
AD
1257 $src = rewrite_relative_url($site_url, $entry->getAttribute('src'));
1258
41bead9b 1259 $local_filename = CACHE_DIR . "/images/" . sha1($src);
3c696512 1260
41bead9b 1261 if ($debug) _debug("cache_media: downloading: $src to $local_filename");
3c696512
AD
1262
1263 if (!file_exists($local_filename)) {
1264 $file_content = fetch_file_contents($src);
1265
41bead9b 1266 if ($file_content && strlen($file_content) > _MIN_CACHE_FILE_SIZE) {
3c696512
AD
1267 file_put_contents($local_filename, $file_content);
1268 }
4a27966e
J
1269 } else {
1270 touch($local_filename);
3c696512 1271 }
3c696512
AD
1272 }
1273 }
3c696512
AD
1274 }
1275
a42c55f0 1276 function expire_error_log($debug) {
e2261e17
AD
1277 if ($debug) _debug("Removing old error log entries...");
1278
1279 if (DB_TYPE == "pgsql") {
a42c55f0 1280 db_query("DELETE FROM ttrss_error_log
e2261e17
AD
1281 WHERE created_at < NOW() - INTERVAL '7 days'");
1282 } else {
a42c55f0 1283 db_query("DELETE FROM ttrss_error_log
e2261e17
AD
1284 WHERE created_at < DATE_SUB(NOW(), INTERVAL 7 DAY)");
1285 }
1286
1287 }
1288
2a91b6ff 1289 function expire_lock_files($debug) {
65465085 1290 //if ($debug) _debug("Removing old lock files...");
2a91b6ff
AD
1291
1292 $num_deleted = 0;
1293
1294 if (is_writable(LOCK_DIRECTORY)) {
1295 $files = glob(LOCK_DIRECTORY . "/*.lock");
1296
1297 if ($files) {
1298 foreach ($files as $file) {
11344971 1299 if (!file_is_locked(basename($file)) && time() - filemtime($file) > 86400*2) {
2a91b6ff
AD
1300 unlink($file);
1301 ++$num_deleted;
1302 }
1303 }
1304 }
1305 }
1306
65465085 1307 if ($debug) _debug("Removed $num_deleted old lock files.");
2a91b6ff
AD
1308 }
1309
3c696512 1310 function expire_cached_files($debug) {
3306daec 1311 foreach (array("simplepie", "images", "export", "upload") as $dir) {
3c696512 1312 $cache_dir = CACHE_DIR . "/$dir";
2c08214a 1313
65465085 1314// if ($debug) _debug("Expiring $cache_dir");
2c08214a 1315
3c696512
AD
1316 $num_deleted = 0;
1317
1318 if (is_writable($cache_dir)) {
1319 $files = glob("$cache_dir/*");
1320
2a91b6ff 1321 if ($files) {
2ab20c31
AD
1322 foreach ($files as $file) {
1323 if (time() - filemtime($file) > 86400*7) {
1324 unlink($file);
3c696512 1325
2ab20c31
AD
1326 ++$num_deleted;
1327 }
3c696512
AD
1328 }
1329 }
2a91b6ff 1330 }
3c696512 1331
65465085 1332 if ($debug) _debug("$cache_dir: removed $num_deleted files.");
3c696512
AD
1333 }
1334 }
2c08214a 1335
a3e0bdcf
AD
1336 /**
1337 * Source: http://www.php.net/manual/en/function.parse-url.php#104527
1338 * Returns the url query as associative array
1339 *
1340 * @param string query
1341 * @return array params
1342 */
1343 function convertUrlQuery($query) {
1344 $queryParts = explode('&', $query);
1345
1346 $params = array();
1347
1348 foreach ($queryParts as $param) {
1349 $item = explode('=', $param);
1350 $params[$item[0]] = $item[1];
1351 }
1352
1353 return $params;
1354 }
92c14e9d 1355
7b55001e 1356 function get_article_filters($filters, $title, $content, $link, $author, $tags, &$matched_rules = false) {
92c14e9d
AD
1357 $matches = array();
1358
1359 foreach ($filters as $filter) {
1360 $match_any_rule = $filter["match_any_rule"];
a3a896a1 1361 $inverse = $filter["inverse"];
92c14e9d
AD
1362 $filter_match = false;
1363
1364 foreach ($filter["rules"] as $rule) {
1365 $match = false;
ffa1bd7b 1366 $reg_exp = str_replace('/', '\/', $rule["reg_exp"]);
a3a896a1 1367 $rule_inverse = $rule["inverse"];
92c14e9d
AD
1368
1369 if (!$reg_exp)
1370 continue;
1371
1372 switch ($rule["type"]) {
1373 case "title":
e3dceca6 1374 $match = @preg_match("/$reg_exp/iu", $title);
92c14e9d
AD
1375 break;
1376 case "content":
d03ae73e
AD
1377 // we don't need to deal with multiline regexps
1378 $content = preg_replace("/[\r\n\t]/", "", $content);
1379
e3dceca6 1380 $match = @preg_match("/$reg_exp/iu", $content);
92c14e9d
AD
1381 break;
1382 case "both":
d03ae73e
AD
1383 // we don't need to deal with multiline regexps
1384 $content = preg_replace("/[\r\n\t]/", "", $content);
1385
e3dceca6 1386 $match = (@preg_match("/$reg_exp/iu", $title) || @preg_match("/$reg_exp/iu", $content));
92c14e9d
AD
1387 break;
1388 case "link":
e3dceca6 1389 $match = @preg_match("/$reg_exp/iu", $link);
92c14e9d
AD
1390 break;
1391 case "author":
e3dceca6 1392 $match = @preg_match("/$reg_exp/iu", $author);
92c14e9d
AD
1393 break;
1394 case "tag":
7b80b5e1 1395 foreach ($tags as $tag) {
e3dceca6 1396 if (@preg_match("/$reg_exp/iu", $tag)) {
7b80b5e1
DK
1397 $match = true;
1398 break;
1399 }
1400 }
92c14e9d
AD
1401 break;
1402 }
1403
a3a896a1
AD
1404 if ($rule_inverse) $match = !$match;
1405
92c14e9d
AD
1406 if ($match_any_rule) {
1407 if ($match) {
1408 $filter_match = true;
1409 break;
1410 }
1411 } else {
1412 $filter_match = $match;
1413 if (!$match) {
1414 break;
1415 }
1416 }
1417 }
1418
a3a896a1
AD
1419 if ($inverse) $filter_match = !$filter_match;
1420
92c14e9d 1421 if ($filter_match) {
557d86fe
AD
1422 if (is_array($matched_rules)) array_push($matched_rules, $rule);
1423
92c14e9d
AD
1424 foreach ($filter["actions"] AS $action) {
1425 array_push($matches, $action);
5e736e45
AD
1426
1427 // if Stop action encountered, perform no further processing
fd3e5e8d 1428 if (isset($action["type"]) && $action["type"] == "stop") return $matches;
92c14e9d
AD
1429 }
1430 }
1431 }
1432
1433 return $matches;
1434 }
1435
1436 function find_article_filter($filters, $filter_name) {
1437 foreach ($filters as $f) {
1438 if ($f["type"] == $filter_name) {
1439 return $f;
1440 };
1441 }
1442 return false;
1443 }
1444
1445 function find_article_filters($filters, $filter_name) {
1446 $results = array();
1447
1448 foreach ($filters as $f) {
1449 if ($f["type"] == $filter_name) {
1450 array_push($results, $f);
1451 };
1452 }
1453 return $results;
1454 }
1455
1456 function calculate_article_score($filters) {
1457 $score = 0;
1458
1459 foreach ($filters as $f) {
1460 if ($f["type"] == "score") {
1461 $score += $f["param"];
1462 };
1463 }
1464 return $score;
1465 }
1466
b24504b1
AD
1467 function labels_contains_caption($labels, $caption) {
1468 foreach ($labels as $label) {
1469 if ($label[1] == $caption) {
1470 return true;
1471 }
1472 }
1473
1474 return false;
1475 }
1476
a42c55f0 1477 function assign_article_to_label_filters($id, $filters, $owner_uid, $article_labels) {
92c14e9d
AD
1478 foreach ($filters as $f) {
1479 if ($f["type"] == "label") {
b24504b1 1480 if (!labels_contains_caption($article_labels, $f["param"])) {
a42c55f0 1481 label_add_article($id, $f["param"], $owner_uid);
b24504b1
AD
1482 }
1483 }
92c14e9d
AD
1484 }
1485 }
87764a50 1486
87d7e850
AD
1487 function make_guid_from_title($title) {
1488 return preg_replace("/[ \"\',.:;]/", "-",
1489 mb_strtolower(strip_tags($title), 'utf-8'));
1490 }
1491
168cf351
AD
1492 function cleanup_counters_cache($debug) {
1493 $result = db_query("DELETE FROM ttrss_counters_cache
1494 WHERE feed_id > 0 AND
1495 (SELECT COUNT(id) FROM ttrss_feeds WHERE
1496 id = feed_id AND
1497 ttrss_counters_cache.owner_uid = ttrss_feeds.owner_uid) = 0");
1498 $frows = db_affected_rows($result);
1499
1500 $result = db_query("DELETE FROM ttrss_cat_counters_cache
1501 WHERE feed_id > 0 AND
1502 (SELECT COUNT(id) FROM ttrss_feed_categories WHERE
1503 id = feed_id AND
1504 ttrss_cat_counters_cache.owner_uid = ttrss_feed_categories.owner_uid) = 0");
1505 $crows = db_affected_rows($result);
1506
7b55001e 1507 if ($debug) _debug("Removed $frows (feeds) $crows (cats) orphaned counter cache entries.");
168cf351
AD
1508 }
1509
5cbd1fe8
AD
1510 function housekeeping_user($owner_uid) {
1511 $tmph = new PluginHost();
1512
1513 load_user_plugins($owner_uid, $tmph);
1514
1515 $tmph->run_hooks(PluginHost::HOOK_HOUSE_KEEPING, "hook_house_keeping", "");
1516 }
1517
e2cf81e2
AD
1518 function housekeeping_common($debug) {
1519 expire_cached_files($debug);
1520 expire_lock_files($debug);
1521 expire_error_log($debug);
1522
1523 $count = update_feedbrowser_cache();
1524 _debug("Feedbrowser updated, $count feeds processed.");
1525
1526 purge_orphans( true);
168cf351 1527 cleanup_counters_cache($debug);
e2cf81e2 1528
9b736a20
AD
1529 //$rc = cleanup_tags( 14, 50000);
1530 //_debug("Cleaned $rc cached tags.");
8e470220 1531
00f22824 1532 PluginHost::getInstance()->run_hooks(PluginHost::HOOK_HOUSE_KEEPING, "hook_house_keeping", "");
e2cf81e2 1533 }
2c08214a 1534?>