X-Git-Url: https://git.wh0rd.org/?a=blobdiff_plain;f=classes%2Frssutils.php;h=b469c47021058c08c10556484a749485eef8097c;hb=81fc862e370a1dfbd3941206fd00076e3cbf0551;hp=696b8604cce4b88b4811af07bab363af99b62acb;hpb=187abfe732fe62cf4b30847665dab30903d00d99;p=tt-rss.git diff --git a/classes/rssutils.php b/classes/rssutils.php old mode 100644 new mode 100755 index 696b8604..b469c470 --- a/classes/rssutils.php +++ b/classes/rssutils.php @@ -7,8 +7,6 @@ class RSSUtils { if ($k != "feed" && isset($v)) { $x = strip_tags(is_array($v) ? implode(",", $v) : $v); - //_debug("$k:" . sha1($x) . ":" . htmlspecialchars($x), true); - $tmp .= sha1("$k:" . sha1($x)); } } @@ -16,6 +14,11 @@ class RSSUtils { return sha1(implode(",", $pluginhost->get_plugin_names()) . $tmp); } + // Strips utf8mb4 characters (i.e. emoji) for mysql + static function strip_utf8mb4($str) { + return preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $str); + } + static function update_feedbrowser_cache() { $pdo = Db::pdo(); @@ -45,8 +48,8 @@ class RSSUtils { if (!$tmph->fetch()) { $tmph = $pdo->prepare("INSERT INTO ttrss_feedbrowser_cache - (feed_url, site_url, title, subscribers) - VALUES + (feed_url, site_url, title, subscribers) + VALUES (?, ?, ?, ?)"); $tmph->execute([$feed_url, $site_url, $title, $subscribers]); @@ -63,7 +66,7 @@ class RSSUtils { } - static function update_daemon_common($limit = DAEMON_FEED_LIMIT, $debug = true) { + static function update_daemon_common($limit = DAEMON_FEED_LIMIT) { $schema_version = get_schema_version(); if ($schema_version != SCHEMA_VERSION) { @@ -140,7 +143,7 @@ class RSSUtils { array_push($feeds_to_update, $line['feed_url']); } - if ($debug) _debug(sprintf("Scheduled %d feeds to update...", count($feeds_to_update))); + Debug::log(sprintf("Scheduled %d feeds to update...", count($feeds_to_update))); // Update last_update_started before actually starting the batch // in order to minimize collision risk for parallel daemon tasks @@ -170,40 +173,52 @@ class RSSUtils { ORDER BY ttrss_feeds.id $query_limit"); foreach ($feeds_to_update as $feed) { - if($debug) _debug("Base feed: $feed"); + Debug::log("Base feed: $feed"); $usth->execute([$feed]); //update_rss_feed($line["id"], true); if ($tline = $usth->fetch()) { - if ($debug) _debug(" => " . $tline["last_updated"] . ", " . $tline["id"] . " " . $tline["owner_uid"]); + Debug::log(" => " . $tline["last_updated"] . ", " . $tline["id"] . " " . $tline["owner_uid"]); if (array_search($tline["owner_uid"], $batch_owners) === FALSE) array_push($batch_owners, $tline["owner_uid"]); $fstarted = microtime(true); - RSSUtils::update_rss_feed($tline["id"], true, false); - _debug_suppress(false); - _debug(sprintf(" %.4f (sec)", microtime(true) - $fstarted)); + try { + RSSUtils::update_rss_feed($tline["id"], true, false); + } catch (PDOException $e) { + Logger::get()->log_error(E_USER_NOTICE, $e->getMessage(), $e->getFile(), $e->getLine(), $e->getTraceAsString()); + + try { + $pdo->rollback(); + } catch (PDOException $e) { + // it doesn't matter if there wasn't actually anything to rollback, PDO Exception can be + // thrown outside of an active transaction during feed update + } + } + + Debug::log(sprintf(" %.4f (sec)", microtime(true) - $fstarted)); ++$nf; } } if ($nf > 0) { - _debug(sprintf("Processed %d feeds in %.4f (sec), %.4f (sec/feed avg)", $nf, + Debug::log(sprintf("Processed %d feeds in %.4f (sec), %.4f (sec/feed avg)", $nf, microtime(true) - $bstarted, (microtime(true) - $bstarted) / $nf)); } foreach ($batch_owners as $owner_uid) { - _debug("Running housekeeping tasks for user $owner_uid..."); + Debug::log("Running housekeeping tasks for user $owner_uid..."); RSSUtils::housekeeping_user($owner_uid); } // Send feed digests by email if needed. - Digest::send_headlines_digests($debug); + if (DIGEST_SUBJECT !== false) + Digest::send_headlines_digests(); return $nf; } @@ -213,24 +228,15 @@ class RSSUtils { $pdo = Db::pdo(); - $sth = $pdo->prepare("SELECT owner_uid,feed_url,auth_pass,auth_login,auth_pass_encrypted + $sth = $pdo->prepare("SELECT owner_uid,feed_url,auth_pass,auth_login FROM ttrss_feeds WHERE id = ?"); $sth->execute([$feed]); if ($row = $sth->fetch()) { $owner_uid = $row["owner_uid"]; - - $auth_pass_encrypted = $row["auth_pass_encrypted"]; - $auth_login = $row["auth_login"]; $auth_pass = $row["auth_pass"]; - - if ($auth_pass_encrypted && function_exists("mcrypt_decrypt")) { - require_once "crypt.php"; - $auth_pass = decrypt_string($auth_pass); - } - $fetch_url = $row["feed_url"]; $pluginhost = new PluginHost(); @@ -304,10 +310,7 @@ class RSSUtils { */ static function update_rss_feed($feed, $no_cache = false) { - $debug_enabled = defined('DAEMON_EXTENDED_DEBUG') || $_REQUEST['xdebug']; - - _debug_suppress(!$debug_enabled); - _debug("start", $debug_enabled); + Debug::log("start", Debug::$LOG_VERBOSE); $pdo = Db::pdo(); @@ -315,7 +318,7 @@ class RSSUtils { $sth->execute([$feed]); if (!$row = $sth->fetch()) { - _debug("feed $feed NOT FOUND/SKIPPED", $debug_enabled); + Debug::log("feed $feed not found, skipping."); user_error("Attempt to update unknown/invalid feed $feed", E_USER_WARNING); return false; } @@ -325,24 +328,23 @@ class RSSUtils { // feed was batch-subscribed or something, we need to get basic info // this is not optimal currently as it fetches stuff separately TODO: optimize if ($title == "[Unknown]") { - _debug("setting basic feed info for $feed..."); + Debug::log("setting basic feed info for $feed..."); RSSUtils::set_basic_feed_info($feed); } $sth = $pdo->prepare("SELECT id,update_interval,auth_login, feed_url,auth_pass,cache_images, mark_unread_on_update, owner_uid, - auth_pass_encrypted, feed_language, - last_modified, - ".SUBSTRING_FOR_DATE."(last_unconditional, 1, 19) AS last_unconditional + auth_pass_encrypted, feed_language, + last_modified, + ".SUBSTRING_FOR_DATE."(last_unconditional, 1, 19) AS last_unconditional FROM ttrss_feeds WHERE id = ?"); $sth->execute([$feed]); - + if ($row = $sth->fetch()) { $owner_uid = $row["owner_uid"]; $mark_unread_on_update = $row["mark_unread_on_update"]; - $auth_pass_encrypted = $row["auth_pass_encrypted"]; $sth = $pdo->prepare("UPDATE ttrss_feeds SET last_update_started = NOW() WHERE id = ?"); @@ -350,16 +352,11 @@ class RSSUtils { $auth_login = $row["auth_login"]; $auth_pass = $row["auth_pass"]; - - if ($auth_pass_encrypted && function_exists("mcrypt_decrypt")) { - require_once "crypt.php"; - $auth_pass = decrypt_string($auth_pass); - } - $stored_last_modified = $row["last_modified"]; $last_unconditional = $row["last_unconditional"]; $cache_images = $row["cache_images"]; $fetch_url = $row["feed_url"]; + $feed_language = mb_strtolower($row["feed_language"]); if (!$feed_language) $feed_language = 'english'; @@ -369,10 +366,9 @@ class RSSUtils { $date_feed_processed = date('Y-m-d H:i'); - $cache_filename = CACHE_DIR . "/simplepie/" . sha1($fetch_url) . ".xml"; + $cache_filename = CACHE_DIR . "/feeds/" . sha1($fetch_url) . ".xml"; $pluginhost = new PluginHost(); - $pluginhost->set_debug($debug_enabled); $user_plugins = get_pref("_ENABLED_PLUGINS", $owner_uid); $pluginhost->load(PLUGINS, PluginHost::KIND_ALL); @@ -395,7 +391,7 @@ class RSSUtils { !$auth_login && !$auth_pass && filemtime($cache_filename) > time() - 30) { - _debug("using local cache [$cache_filename].", $debug_enabled); + Debug::log("using local cache [$cache_filename].", Debug::$LOG_VERBOSE); @$feed_data = file_get_contents($cache_filename); @@ -404,28 +400,28 @@ class RSSUtils { } } else { - _debug("local cache will not be used for this feed", $debug_enabled); + Debug::log("local cache will not be used for this feed", Debug::$LOG_VERBOSE); } global $fetch_last_modified; // fetch feed from source if (!$feed_data) { - _debug("last unconditional update request: $last_unconditional"); + Debug::log("last unconditional update request: $last_unconditional", Debug::$LOG_VERBOSE); if (ini_get("open_basedir") && function_exists("curl_init")) { - _debug("not using CURL due to open_basedir restrictions"); + Debug::log("not using CURL due to open_basedir restrictions", Debug::$LOG_VERBOSE); } - if (time() - strtotime($last_unconditional) > MAX_CONDITIONAL_INTERVAL) { - _debug("maximum allowed interval for conditional requests exceeded, forcing refetch"); + if (time() - strtotime($last_unconditional) > MAX_CONDITIONAL_INTERVAL) { + Debug::log("maximum allowed interval for conditional requests exceeded, forcing refetch", Debug::$LOG_VERBOSE); - $force_refetch = true; - } else { - _debug("stored last modified for conditional request: $stored_last_modified", $debug_enabled); - } + $force_refetch = true; + } else { + Debug::log("stored last modified for conditional request: $stored_last_modified", Debug::$LOG_VERBOSE); + } - _debug("fetching [$fetch_url] (force_refetch: $force_refetch)...", $debug_enabled); + Debug::log("fetching [$fetch_url] (force_refetch: $force_refetch)...", Debug::$LOG_VERBOSE); $feed_data = fetch_file_contents([ "url" => $fetch_url, @@ -445,8 +441,8 @@ class RSSUtils { $feed_data = trim($feed_data); - _debug("fetch done.", $debug_enabled); - _debug("source last modified: " . $fetch_last_modified, $debug_enabled); + Debug::log("fetch done.", Debug::$LOG_VERBOSE); + Debug::log("source last modified: " . $fetch_last_modified, Debug::$LOG_VERBOSE); if ($feed_data && $fetch_last_modified != $stored_last_modified) { $sth = $pdo->prepare("UPDATE ttrss_feeds SET last_modified = ? WHERE id = ?"); @@ -454,11 +450,11 @@ class RSSUtils { } // cache vanilla feed data for re-use - if ($feed_data && !$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) { + if ($feed_data && !$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/feeds")) { $new_rss_hash = sha1($feed_data); if ($new_rss_hash != $rss_hash) { - _debug("saving $cache_filename", $debug_enabled); + Debug::log("saving $cache_filename", Debug::$LOG_VERBOSE); @file_put_contents($cache_filename, $feed_data); } } @@ -468,13 +464,13 @@ class RSSUtils { global $fetch_last_error; global $fetch_last_error_code; - _debug("unable to fetch: $fetch_last_error [$fetch_last_error_code]", $debug_enabled); + Debug::log("unable to fetch: $fetch_last_error [$fetch_last_error_code]", Debug::$LOG_VERBOSE); // If-Modified-Since if ($fetch_last_error_code != 304) { $error_message = $fetch_last_error; } else { - _debug("source claims data not modified, nothing to do.", $debug_enabled); + Debug::log("source claims data not modified, nothing to do.", Debug::$LOG_VERBOSE); $error_message = ""; } @@ -492,15 +488,13 @@ class RSSUtils { $rss = new FeedParser($feed_data); $rss->init(); - $feed = $feed; - if (!$rss->error()) { // We use local pluginhost here because we need to load different per-user feed plugins $pluginhost->run_hooks(PluginHost::HOOK_FEED_PARSED, "hook_feed_parsed", $rss); - _debug("language: $feed_language", $debug_enabled); - _debug("processing feed data...", $debug_enabled); + Debug::log("language: $feed_language", Debug::$LOG_VERBOSE); + Debug::log("processing feed data...", Debug::$LOG_VERBOSE); if (DB_TYPE == "pgsql") { $favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'"; @@ -524,8 +518,8 @@ class RSSUtils { $site_url = mb_substr(rewrite_relative_url($fetch_url, $rss->get_link()), 0, 245); - _debug("site_url: $site_url", $debug_enabled); - _debug("feed_title: " . $rss->get_title(), $debug_enabled); + Debug::log("site_url: $site_url", Debug::$LOG_VERBOSE); + Debug::log("feed_title: " . $rss->get_title(), Debug::$LOG_VERBOSE); if ($favicon_needs_check || $force_refetch) { @@ -535,7 +529,7 @@ class RSSUtils { $favicon_file = ICONS_DIR . "/$feed.ico"; $favicon_modified = @filemtime($favicon_file); - _debug("checking favicon...", $debug_enabled); + Debug::log("checking favicon...", Debug::$LOG_VERBOSE); RSSUtils::check_feed_favicon($site_url, $feed); $favicon_modified_new = @filemtime($favicon_file); @@ -556,7 +550,7 @@ class RSSUtils { $favicon_colorstring = ",favicon_avg_color = " . $pdo->quote($favicon_color); } else if ($favicon_avg_color == 'fail') { - _debug("floicon failed on this file, not trying to recalculate avg color", $debug_enabled); + Debug::log("floicon failed on this file, not trying to recalculate avg color", Debug::$LOG_VERBOSE); } $sth = $pdo->prepare("UPDATE ttrss_feeds SET favicon_last_checked = NOW() @@ -564,20 +558,20 @@ class RSSUtils { $sth->execute([$feed]); } - _debug("loading filters & labels...", $debug_enabled); + Debug::log("loading filters & labels...", Debug::$LOG_VERBOSE); $filters = load_filters($feed, $owner_uid); - if ($debug_enabled) { - print_r($filters); - } + if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { + print_r($filters); + } - _debug("" . count($filters) . " filters loaded.", $debug_enabled); + Debug::log("" . count($filters) . " filters loaded.", Debug::$LOG_VERBOSE); $items = $rss->get_items(); if (!is_array($items)) { - _debug("no articles found.", $debug_enabled); + Debug::log("no articles found.", Debug::$LOG_VERBOSE); $sth = $pdo->prepare("UPDATE ttrss_feeds SET last_updated = NOW(), last_unconditional = NOW(), last_error = '' WHERE id = ?"); @@ -586,19 +580,19 @@ class RSSUtils { return true; // no articles } - _debug("processing articles...", $debug_enabled); + Debug::log("processing articles...", Debug::$LOG_VERBOSE); $tstart = time(); foreach ($items as $item) { $pdo->beginTransaction(); - if ($_REQUEST['xdebug'] == 3) { + if (Debug::get_loglevel() >= 3) { print_r($item); } if (ini_get("max_execution_time") > 0 && time() - $tstart >= ini_get("max_execution_time") * 0.7) { - _debug("looks like there's too many articles to process at once, breaking out", $debug_enabled); + Debug::log("looks like there's too many articles to process at once, breaking out", Debug::$LOG_VERBOSE); $pdo->commit(); break; } @@ -616,11 +610,11 @@ class RSSUtils { $entry_guid_hashed = 'SHA1:' . sha1($entry_guid); - _debug("guid $entry_guid / $entry_guid_hashed", $debug_enabled); + Debug::log("guid $entry_guid / $entry_guid_hashed", Debug::$LOG_VERBOSE); $entry_timestamp = strip_tags($item->get_date()); - _debug("orig date: " . $item->get_date(), $debug_enabled); + Debug::log("orig date: " . $item->get_date(), Debug::$LOG_VERBOSE); if ($entry_timestamp == -1 || !$entry_timestamp || $entry_timestamp > time()) { $entry_timestamp = time(); @@ -628,21 +622,24 @@ class RSSUtils { $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); - _debug("date $entry_timestamp [$entry_timestamp_fmt]", $debug_enabled); + Debug::log("date $entry_timestamp [$entry_timestamp_fmt]", Debug::$LOG_VERBOSE); $entry_title = strip_tags($item->get_title()); $entry_link = rewrite_relative_url($site_url, $item->get_link()); - _debug("title $entry_title", $debug_enabled); - _debug("link $entry_link", $debug_enabled); + $entry_language = mb_substr(trim($item->get_language()), 0, 2); + + Debug::log("title $entry_title", Debug::$LOG_VERBOSE); + Debug::log("link $entry_link", Debug::$LOG_VERBOSE); + Debug::log("language $entry_language", Debug::$LOG_VERBOSE); if (!$entry_title) $entry_title = date("Y-m-d H:i:s", $entry_timestamp);; $entry_content = $item->get_content(); if (!$entry_content) $entry_content = $item->get_description(); - if ($_REQUEST["xdebug"] == 2) { + if (Debug::get_loglevel() >= 3) { print "content: "; print htmlspecialchars($entry_content); print "\n"; @@ -654,9 +651,9 @@ class RSSUtils { $entry_author = strip_tags($item->get_author()); $entry_guid = mb_substr($entry_guid, 0, 245); - _debug("author $entry_author", $debug_enabled); - _debug("num_comments: $num_comments", $debug_enabled); - _debug("looking for tags...", $debug_enabled); + Debug::log("author $entry_author", Debug::$LOG_VERBOSE); + Debug::log("num_comments: $num_comments", Debug::$LOG_VERBOSE); + Debug::log("looking for tags...", Debug::$LOG_VERBOSE); // parse entries into tags @@ -672,12 +669,16 @@ class RSSUtils { $entry_tags = array_unique($additional_tags); - for ($i = 0; $i < count($entry_tags); $i++) + for ($i = 0; $i < count($entry_tags); $i++) { $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8'); - _debug("tags found: " . join(",", $entry_tags), $debug_enabled); + // we don't support numeric tags, let's prefix them + if (is_numeric($entry_tags[$i])) $entry_tags[$i] = 't:' . $entry_tags[$i]; + } + + Debug::log("tags found: " . join(",", $entry_tags), Debug::$LOG_VERBOSE); - _debug("done collecting data.", $debug_enabled); + Debug::log("done collecting data.", Debug::$LOG_VERBOSE); $sth = $pdo->prepare("SELECT id, content_hash, lang FROM ttrss_entries WHERE guid = ? OR guid = ?"); @@ -687,7 +688,6 @@ class RSSUtils { $base_entry_id = $row["id"]; $entry_stored_hash = $row["content_hash"]; $article_labels = Article::get_article_labels($base_entry_id, $owner_uid); - $entry_language = $row["lang"]; $existing_tags = Article::get_article_tags($base_entry_id, $owner_uid); $entry_tags = array_unique(array_merge($entry_tags, $existing_tags)); @@ -695,7 +695,6 @@ class RSSUtils { $base_entry_id = false; $entry_stored_hash = ""; $article_labels = array(); - $entry_language = ""; } $article = array("owner_uid" => $owner_uid, // read only @@ -720,10 +719,10 @@ class RSSUtils { $entry_plugin_data = ""; $entry_current_hash = RSSUtils::calculate_article_hash($article, $pluginhost); - _debug("article hash: $entry_current_hash [stored=$entry_stored_hash]", $debug_enabled); + Debug::log("article hash: $entry_current_hash [stored=$entry_stored_hash]", Debug::$LOG_VERBOSE); if ($entry_current_hash == $entry_stored_hash && !isset($_REQUEST["force_rehash"])) { - _debug("stored article seems up to date [IID: $base_entry_id], updating timestamp only", $debug_enabled); + Debug::log("stored article seems up to date [IID: $base_entry_id], updating timestamp only", Debug::$LOG_VERBOSE); // we keep encountering the entry in feeds, so we need to // update date_updated column so that we don't get horrible @@ -738,34 +737,33 @@ class RSSUtils { continue; } - _debug("hash differs, applying plugin filters:", $debug_enabled); + Debug::log("hash differs, applying plugin filters:", Debug::$LOG_VERBOSE); foreach ($pluginhost->get_hooks(PluginHost::HOOK_ARTICLE_FILTER) as $plugin) { - _debug("... " . get_class($plugin), $debug_enabled); + Debug::log("... " . get_class($plugin), Debug::$LOG_VERBOSE); $start = microtime(true); $article = $plugin->hook_article_filter($article); - _debug("=== " . sprintf("%.4f (sec)", microtime(true) - $start), $debug_enabled); + Debug::log(sprintf("=== %.4f (sec)", microtime(true) - $start), Debug::$LOG_VERBOSE); $entry_plugin_data .= mb_strtolower(get_class($plugin)) . ","; } - if ($_REQUEST["xdebug"] == 2) { + if (Debug::get_loglevel() >= 3) { print "processed content: "; print htmlspecialchars($article["content"]); print "\n"; } - _debug("plugin data: $entry_plugin_data", $debug_enabled); + Debug::log("plugin data: $entry_plugin_data", Debug::$LOG_VERBOSE); // Workaround: 4-byte unicode requires utf8mb4 in MySQL. See https://tt-rss.org/forum/viewtopic.php?f=1&t=3377&p=20077#p20077 - if (DB_TYPE == "mysql") { + if (DB_TYPE == "mysql" && MYSQL_CHARSET != "UTF8MB4") { foreach ($article as $k => $v) { - // i guess we'll have to take the risk of 4byte unicode labels & tags here if (is_string($article[$k])) { - $article[$k] = preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $v); + $article[$k] = RSSUtils::strip_utf8mb4($v); } } } @@ -778,14 +776,14 @@ class RSSUtils { $article["content"], $article["link"], $article["author"], $article["tags"], $matched_rules); - if ($debug_enabled) { - _debug("matched filter rules: ", $debug_enabled); + if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { + Debug::log("matched filter rules: ", Debug::$LOG_VERBOSE); if (count($matched_rules) != 0) { print_r($matched_rules); } - _debug("filter actions: ", $debug_enabled); + Debug::log("filter actions: ", Debug::$LOG_VERBOSE); if (count($article_filters) != 0) { print_r($article_filters); @@ -796,7 +794,7 @@ class RSSUtils { $plugin_filter_actions = $pluginhost->get_filter_actions(); if (count($plugin_filter_names) > 0) { - _debug("applying plugin filter actions...", $debug_enabled); + Debug::log("applying plugin filter actions...", Debug::$LOG_VERBOSE); foreach ($plugin_filter_names as $pfn) { list($pfclass,$pfaction) = explode(":", $pfn["param"]); @@ -804,18 +802,18 @@ class RSSUtils { if (isset($plugin_filter_actions[$pfclass])) { $plugin = $pluginhost->get_plugin($pfclass); - _debug("... $pfclass: $pfaction", $debug_enabled); + Debug::log("... $pfclass: $pfaction", Debug::$LOG_VERBOSE); if ($plugin) { $start = microtime(true); $article = $plugin->hook_article_filter_action($article, $pfaction); - _debug("=== " . sprintf("%.4f (sec)", microtime(true) - $start), $debug_enabled); + Debug::log(sprintf("=== %.4f (sec)"), Debug::$LOG_VERBOSE); } else { - _debug("??? $pfclass: plugin object not found."); + Debug::log("??? $pfclass: plugin object not found.", Debug::$LOG_VERBOSE); } } else { - _debug("??? $pfclass: filter plugin not registered."); + Debug::log("??? $pfclass: filter plugin not registered.", Debug::$LOG_VERBOSE); } } } @@ -830,18 +828,18 @@ class RSSUtils { $entry_score_modifier = (int) $article["score_modifier"]; $entry_language = $article["language"]; - if ($debug_enabled) { - _debug("article labels:", $debug_enabled); + if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { + Debug::log("article labels:", Debug::$LOG_VERBOSE); if (count($article_labels) != 0) { print_r($article_labels); } } - _debug("force catchup: $entry_force_catchup"); + Debug::log("force catchup: $entry_force_catchup", Debug::$LOG_VERBOSE); if ($cache_images && is_writable(CACHE_DIR . '/images')) - RSSUtils::cache_media($entry_content, $site_url, $debug_enabled); + RSSUtils::cache_media($entry_content, $site_url); $csth = $pdo->prepare("SELECT id FROM ttrss_entries WHERE guid = ? OR guid = ?"); @@ -849,13 +847,13 @@ class RSSUtils { if (!$row = $csth->fetch()) { - _debug("base guid [$entry_guid or $entry_guid_hashed] not found, creating...", $debug_enabled); + Debug::log("base guid [$entry_guid or $entry_guid_hashed] not found, creating...", Debug::$LOG_VERBOSE); // base post entry does not exist, create it $usth = $pdo->prepare( "INSERT INTO ttrss_entries - (title, + (title, guid, link, updated, @@ -879,14 +877,14 @@ class RSSUtils { $entry_guid_hashed, $entry_link, $entry_timestamp_fmt, - $entry_content, + "$entry_content", $entry_current_hash, $date_feed_processed, $entry_comments, (int)$num_comments, $entry_plugin_data, - $entry_language, - $entry_author]); + "$entry_language", + "$entry_author"]); } @@ -897,7 +895,7 @@ class RSSUtils { if ($row = $csth->fetch()) { - _debug("base guid found, checking for user record", $debug_enabled); + Debug::log("base guid found, checking for user record", Debug::$LOG_VERBOSE); $ref_id = $row['id']; $entry_ref_id = $ref_id; @@ -909,7 +907,7 @@ class RSSUtils { $score = RSSUtils::calculate_article_score($article_filters) + $entry_score_modifier; - _debug("initial score: $score [including plugin modifier: $entry_score_modifier]", $debug_enabled); + Debug::log("initial score: $score [including plugin modifier: $entry_score_modifier]", Debug::$LOG_VERBOSE); // check for user post link to main table @@ -922,17 +920,17 @@ class RSSUtils { $entry_ref_id = $row["ref_id"]; $entry_int_id = $row["int_id"]; - _debug("user record FOUND: RID: $entry_ref_id, IID: $entry_int_id", $debug_enabled); + Debug::log("user record FOUND: RID: $entry_ref_id, IID: $entry_int_id", Debug::$LOG_VERBOSE); } else { - - _debug("user record not found, creating...", $debug_enabled); + + Debug::log("user record not found, creating...", Debug::$LOG_VERBOSE); if ($score >= -500 && !RSSUtils::find_article_filter($article_filters, 'catchup') && !$entry_force_catchup) { $unread = 1; $last_read_qpart = null; } else { $unread = 0; - $last_read_qpart = 'NOW()'; + $last_read_qpart = date("Y-m-d H:i"); // we can't use NOW() here because it gets quoted } if (RSSUtils::find_article_filter($article_filters, 'mark') || $score > 1000) { @@ -947,18 +945,18 @@ class RSSUtils { $published = 0; } - $last_marked = ($marked == 'true') ? 'NOW()' : null; - $last_published = ($published == 'true') ? 'NOW()' : null; + $last_marked = ($marked == 1) ? 'NOW()' : 'NULL'; + $last_published = ($published == 1) ? 'NOW()' : 'NULL'; $sth = $pdo->prepare( "INSERT INTO ttrss_user_entries (ref_id, owner_uid, feed_id, unread, last_read, marked, published, score, tag_cache, label_cache, uuid, last_marked, last_published) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, '', '', '', ?, ?)"); + VALUES (?, ?, ?, ?, ?, ?, ?, ?, '', '', '', ".$last_marked.", ".$last_published.")"); $sth->execute([$ref_id, $owner_uid, $feed, $unread, $last_read_qpart, $marked, - $published, $score, $last_marked, $last_published]); + $published, $score]); $sth = $pdo->prepare("SELECT int_id FROM ttrss_user_entries WHERE ref_id = ? AND owner_uid = ? AND @@ -970,32 +968,42 @@ class RSSUtils { $entry_int_id = $row['int_id']; } - _debug("resulting RID: $entry_ref_id, IID: $entry_int_id", $debug_enabled); + Debug::log("resulting RID: $entry_ref_id, IID: $entry_int_id", Debug::$LOG_VERBOSE); - if (DB_TYPE == "pgsql") { - $tsvector_combined = mb_substr($entry_title . ' ' . strip_tags(str_replace('<', ' <', $entry_content)), - 0, 1000000); - - $tsvector_qpart = "tsvector_combined = to_tsvector('$feed_language', ".$pdo->quote($tsvector_combined)."),"; - - } else { + if (DB_TYPE == "pgsql") + $tsvector_qpart = "tsvector_combined = to_tsvector(:ts_lang, :ts_content),"; + else $tsvector_qpart = ""; - } $sth = $pdo->prepare("UPDATE ttrss_entries - SET title = ?, - content = ?, - content_hash = ?, - updated = ?, + SET title = :title, $tsvector_qpart - num_comments = ?, - plugin_data = ?, - author = ?, - lang = ? - WHERE id = ?"); + content = :content, + content_hash = :content_hash, + updated = :updated, + date_updated = NOW(), + num_comments = :num_comments, + plugin_data = :plugin_data, + author = :author, + lang = :lang + WHERE id = :id"); + + $params = [":title" => $entry_title, + ":content" => "$entry_content", + ":content_hash" => $entry_current_hash, + ":updated" => $entry_timestamp_fmt, + ":num_comments" => (int)$num_comments, + ":plugin_data" => $entry_plugin_data, + ":author" => "$entry_author", + ":lang" => $entry_language, + ":id" => $ref_id]; - $sth->execute([$entry_title, $entry_content, $entry_current_hash, $entry_timestamp_fmt, - (int)$num_comments, $entry_plugin_data, $entry_author, $entry_language, $ref_id]); + if (DB_TYPE == "pgsql") { + $params[":ts_lang"] = $feed_language; + $params[":ts_content"] = mb_substr(strip_tags($entry_title . " " . $entry_content), 0, 900000); + } + + $sth->execute($params); // update aux data $sth = $pdo->prepare("UPDATE ttrss_user_entries @@ -1003,7 +1011,7 @@ class RSSUtils { $sth->execute([$score, $ref_id]); if ($mark_unread_on_update) { - _debug("article updated, marking unread as requested.", $debug_enabled); + Debug::log("article updated, marking unread as requested.", Debug::$LOG_VERBOSE); $sth = $pdo->prepare("UPDATE ttrss_user_entries SET last_read = null, unread = true WHERE ref_id = ?"); @@ -1011,18 +1019,18 @@ class RSSUtils { } } - _debug("assigning labels [other]...", $debug_enabled); + Debug::log("assigning labels [other]...", Debug::$LOG_VERBOSE); foreach ($article_labels as $label) { Labels::add_article($entry_ref_id, $label[1], $owner_uid); } - _debug("assigning labels [filters]...", $debug_enabled); + Debug::log("assigning labels [filters]...", Debug::$LOG_VERBOSE); RSSUtils::assign_article_to_label_filters($entry_ref_id, $article_filters, $owner_uid, $article_labels); - _debug("looking for enclosures...", $debug_enabled); + Debug::log("looking for enclosures...", Debug::$LOG_VERBOSE); // enclosures @@ -1035,20 +1043,30 @@ class RSSUtils { $e_item = array( rewrite_relative_url($site_url, $e->link), $e->type, $e->length, $e->title, $e->width, $e->height); + + // Yet another episode of "mysql utf8_general_ci is gimped" + if (DB_TYPE == "mysql" && MYSQL_CHARSET != "UTF8MB4") { + for ($i = 0; $i < count($e_item); $i++) { + if (is_string($e_item[$i])) { + $e_item[$i] = RSSUtils::strip_utf8mb4($e_item[$i]); + } + } + } + array_push($enclosures, $e_item); } } if ($cache_images && is_writable(CACHE_DIR . '/images')) - RSSUtils::cache_enclosures($enclosures, $site_url, $debug_enabled); + RSSUtils::cache_enclosures($enclosures, $site_url); - if ($debug_enabled) { - _debug("article enclosures:", $debug_enabled); + if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { + Debug::log("article enclosures:", Debug::$LOG_VERBOSE); print_r($enclosures); } $esth = $pdo->prepare("SELECT id FROM ttrss_enclosures - WHERE content_url = ? AND post_id = ?"); + WHERE content_url = ? AND content_type = ? AND post_id = ?"); $usth = $pdo->prepare("INSERT INTO ttrss_enclosures (content_url, content_type, title, duration, post_id, width, height) VALUES @@ -1062,7 +1080,7 @@ class RSSUtils { $enc_width = intval($enc[4]); $enc_height = intval($enc[5]); - $esth->execute([$enc_url, $entry_ref_id]); + $esth->execute([$enc_url, $enc_type, $entry_ref_id]); if (!$esth->fetch()) { $usth->execute([$enc_url, $enc_type, (string)$enc_title, $enc_dur, $entry_ref_id, $enc_width, $enc_height]); @@ -1102,8 +1120,8 @@ class RSSUtils { $filtered_tags = array_unique($filtered_tags); - if ($debug_enabled) { - _debug("filtered article tags:", $debug_enabled); + if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { + Debug::log("filtered article tags:", Debug::$LOG_VERBOSE); print_r($filtered_tags); } @@ -1146,14 +1164,14 @@ class RSSUtils { $tsth->execute([$tags_str, $entry_ref_id, $owner_uid]); } - _debug("article processed", $debug_enabled); + Debug::log("article processed", Debug::$LOG_VERBOSE); $pdo->commit(); } - _debug("purging feed...", $debug_enabled); + Debug::log("purging feed...", Debug::$LOG_VERBOSE); - purge_feed($feed, 0, $debug_enabled); + purge_feed($feed, 0); $sth = $pdo->prepare("UPDATE ttrss_feeds SET last_updated = NOW(), last_unconditional = NOW(), last_error = '' WHERE id = ?"); @@ -1163,11 +1181,11 @@ class RSSUtils { $error_msg = mb_substr($rss->error(), 0, 245); - _debug("fetch error: $error_msg", $debug_enabled); + Debug::log("fetch error: $error_msg", Debug::$LOG_VERBOSE); if (count($rss->errors()) > 1) { foreach ($rss->errors() as $error) { - _debug("+ $error"); + Debug::log("+ $error", Debug::$LOG_VERBOSE); } } @@ -1179,12 +1197,12 @@ class RSSUtils { return false; } - _debug("done", $debug_enabled); + Debug::log("done", Debug::$LOG_VERBOSE); return true; } - static function cache_enclosures($enclosures, $site_url, $debug) { + static function cache_enclosures($enclosures, $site_url) { foreach ($enclosures as $enc) { if (preg_match("/(image|audio|video)/", $enc[1])) { @@ -1193,7 +1211,7 @@ class RSSUtils { $local_filename = CACHE_DIR . "/images/" . sha1($src); - if ($debug) _debug("cache_enclosures: downloading: $src to $local_filename"); + Debug::log("cache_enclosures: downloading: $src to $local_filename", Debug::$LOG_VERBOSE); if (!file_exists($local_filename)) { $file_content = fetch_file_contents($src); @@ -1201,14 +1219,14 @@ class RSSUtils { if ($file_content && strlen($file_content) > MIN_CACHE_FILE_SIZE) { file_put_contents($local_filename, $file_content); } - } else { + } else if (is_writable($local_filename)) { touch($local_filename); } } } } - static function cache_media($html, $site_url, $debug) { + static function cache_media($html, $site_url) { libxml_use_internal_errors(true); $charset_hack = ' @@ -1227,23 +1245,25 @@ class RSSUtils { $local_filename = CACHE_DIR . "/images/" . sha1($src); - if ($debug) _debug("cache_media: downloading: $src to $local_filename"); + Debug::log("cache_media: checking $src", Debug::$LOG_VERBOSE); if (!file_exists($local_filename)) { + Debug::log("cache_media: downloading: $src to $local_filename", Debug::$LOG_VERBOSE); + $file_content = fetch_file_contents($src); if ($file_content && strlen($file_content) > MIN_CACHE_FILE_SIZE) { file_put_contents($local_filename, $file_content); } - } else { + } else if (is_writable($local_filename)) { touch($local_filename); } } } } - static function expire_error_log($debug) { - if ($debug) _debug("Removing old error log entries..."); + static function expire_error_log() { + Debug::log("Removing old error log entries..."); $pdo = Db::pdo(); @@ -1256,8 +1276,8 @@ class RSSUtils { } } - static function expire_lock_files($debug) { - //if ($debug) _debug("Removing old lock files..."); + static function expire_lock_files() { + Debug::log("Removing old lock files...", Debug::$LOG_VERBOSE); $num_deleted = 0; @@ -1274,14 +1294,14 @@ class RSSUtils { } } - if ($debug) _debug("Removed $num_deleted old lock files."); + Debug::log("Removed $num_deleted old lock files."); } - static function expire_cached_files($debug) { - foreach (array("simplepie", "images", "export", "upload") as $dir) { + static function expire_cached_files() { + foreach (array("feeds", "images", "export", "upload") as $dir) { $cache_dir = CACHE_DIR . "/$dir"; -// if ($debug) _debug("Expiring $cache_dir"); + Debug::log("Expiring $cache_dir", Debug::$LOG_VERBOSE); $num_deleted = 0; @@ -1299,7 +1319,7 @@ class RSSUtils { } } - if ($debug) _debug("$cache_dir: removed $num_deleted files."); + Debug::log("$cache_dir: removed $num_deleted files."); } } @@ -1459,7 +1479,7 @@ class RSSUtils { mb_strtolower(strip_tags($title), 'utf-8')); } - static function cleanup_counters_cache($debug) { + static function cleanup_counters_cache() { $pdo = Db::pdo(); $res = $pdo->query("DELETE FROM ttrss_counters_cache @@ -1478,7 +1498,7 @@ class RSSUtils { $crows = $res->rowCount(); - if ($debug) _debug("Removed $frows (feeds) $crows (cats) orphaned counter cache entries."); + Debug::log("Removed $frows (feeds) $crows (cats) orphaned counter cache entries."); } static function housekeeping_user($owner_uid) { @@ -1489,19 +1509,16 @@ class RSSUtils { $tmph->run_hooks(PluginHost::HOOK_HOUSE_KEEPING, "hook_house_keeping", ""); } - static function housekeeping_common($debug) { - RSSUtils::expire_cached_files($debug); - RSSUtils::expire_lock_files($debug); - RSSUtils::expire_error_log($debug); + static function housekeeping_common() { + RSSUtils::expire_cached_files(); + RSSUtils::expire_lock_files(); + RSSUtils::expire_error_log(); $count = RSSUtils::update_feedbrowser_cache(); - _debug("Feedbrowser updated, $count feeds processed."); - - Article::purge_orphans( true); - RSSUtils::cleanup_counters_cache($debug); + Debug::log("Feedbrowser updated, $count feeds processed."); - //$rc = cleanup_tags( 14, 50000); - //_debug("Cleaned $rc cached tags."); + Article::purge_orphans(); + RSSUtils::cleanup_counters_cache(); PluginHost::getInstance()->run_hooks(PluginHost::HOOK_HOUSE_KEEPING, "hook_house_keeping", ""); }