more fts stuff for simple index

[tt-rss.git] / include / rssfuncs.php

diff --git a/include/rssfuncs.php b/include/rssfuncs.php
index dcf86c1af65ef708c100e9ea5b738ceb2c38b27f..5ebddf9eff4ef609ef46ae685e89ec29b4beda3f 100644 (file)
--- a/include/rssfuncs.php
+++ b/include/rssfuncs.php
@@ -291,6 +291,17 @@
                _debug_suppress(!$debug_enabled);
                _debug("start", $debug_enabled);
 
+               $result = db_query("SELECT title FROM ttrss_feeds
+                       WHERE id = '$feed'");
+               $title = db_fetch_result($result, 0, "title");
+
+               // feed was batch-subscribed or something, we need to get basic info
+               // this is not optimal currently as it fetches stuff separately TODO: optimize
+               if ($title == "[Unknown]") {
+                       _debug("setting basic feed info for $feed...");
+                       set_basic_feed_info($feed);
+               }
+
                $result = db_query("SELECT id,update_interval,auth_login,
                        feed_url,auth_pass,cache_images,
                        mark_unread_on_update, owner_uid,
@@ -443,13 +454,6 @@
                        $rss->init();
                }
 
-               if (DETECT_ARTICLE_LANGUAGE) {
-                       require_once "lib/languagedetect/LanguageDetect.php";
-
-                       $lang = new Text_LanguageDetect();
-                       $lang->setNameMode(2);
-               }
-
 //             print_r($rss);
 
                $feed = db_escape_string($feed);
@@ -588,11 +592,18 @@
 
                        _debug("processing articles...", $debug_enabled);
 
+                       $tstart = time();
+
                        foreach ($items as $item) {
                                if ($_REQUEST['xdebug'] == 3) {
                                        print_r($item);
                                }
 
+                               if (ini_get("max_execution_time") > 0 && time() - $tstart >= ini_get("max_execution_time") * 0.7) {
+                                       _debug("looks like there's too many articles to process at once, breaking out", $debug_enabled);
+                                       break;
+                               }
+
                                $entry_guid = $item->get_id();
                                if (!$entry_guid) $entry_guid = $item->get_link();
                                if (!$entry_guid) $entry_guid = make_guid_from_title($item->get_title());
@@ -638,21 +649,6 @@
                                        print "\n";
                                }
 
-                               $entry_language = "";
-
-                               if (DETECT_ARTICLE_LANGUAGE) {
-                                       $entry_language = $lang->detect($entry_title . " " . $entry_content, 1);
-
-                                       if (count($entry_language) > 0) {
-                                               $possible = array_keys($entry_language);
-                                               $entry_language = $possible[0];
-
-                                               _debug("detected language: $entry_language", $debug_enabled);
-                                       } else {
-                                               $entry_language = "";
-                                       }
-                               }
-
                                $entry_comments = $item->get_comments_url();
                                $entry_author = $item->get_author();
 
@@ -688,17 +684,19 @@
 
                                _debug("done collecting data.", $debug_enabled);
 
-                               $result = db_query("SELECT id, content_hash FROM ttrss_entries
+                               $result = db_query("SELECT id, content_hash, lang FROM ttrss_entries
                                        WHERE guid = '".db_escape_string($entry_guid)."' OR guid = '$entry_guid_hashed'");
 
                                if (db_num_rows($result) != 0) {
                                        $base_entry_id = db_fetch_result($result, 0, "id");
                                        $entry_stored_hash = db_fetch_result($result, 0, "content_hash");
                                        $article_labels = get_article_labels($base_entry_id, $owner_uid);
+                                       $entry_language = db_fetch_result($result, 0, "lang");
                                } else {
                                        $base_entry_id = false;
                                        $entry_stored_hash = "";
                                        $article_labels = array();
+                                       $entry_language = "";
                                }
 
                                $article = array("owner_uid" => $owner_uid, // read only
@@ -712,7 +710,7 @@
                                        "author" => $entry_author,
                                        "force_catchup" => false, // ugly hack for the time being
                                        "score_modifier" => 0, // no previous value, plugin should recalculate score modifier based on content if needed
-                                       "language" => $entry_language, // read only
+                                       "language" => $entry_language,
                                        "feed" => array("id" => $feed,
                                                "fetch_url" => $fetch_url,
                                                "site_url" => $site_url)
@@ -763,7 +761,11 @@
                                // Workaround: 4-byte unicode requires utf8mb4 in MySQL. See https://tt-rss.org/forum/viewtopic.php?f=1&t=3377&p=20077#p20077
                                if (DB_TYPE == "mysql") {
                                        foreach ($article as $k => $v) {
-                                               $article[$k] = preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $v);
+
+                                               // i guess we'll have to take the risk of 4byte unicode labels & tags here
+                                               if (!is_array($article[$k])) {
+                                                       $article[$k] = preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $v);
+                                               }
                                        }
                                }
 
@@ -776,6 +778,7 @@
                                $entry_force_catchup = $article["force_catchup"];
                                $article_labels = $article["labels"];
                                $entry_score_modifier = (int) $article["score_modifier"];
+                               $entry_language = db_escape_string($article["language"]);
 
                                if ($debug_enabled) {
                                        _debug("article labels:", $debug_enabled);
@@ -981,11 +984,22 @@
 
                                        _debug("RID: $entry_ref_id, IID: $entry_int_id", $debug_enabled);
 
+                                       if (DB_TYPE == "pgsql") {
+                                               $tsvector_combined = db_escape_string(mb_substr($entry_title . ' ' . strip_tags($entry_content),
+                                                       0, 1000000));
+
+                                               $tsvector_qpart = "tsvector_combined = to_tsvector('simple', '$tsvector_combined'),";
+
+                                       } else {
+                                               $tsvector_qpart = "";
+                                       }
+
                                        db_query("UPDATE ttrss_entries
                                                SET title = '$entry_title',
                                                        content = '$entry_content',
                                                        content_hash = '$entry_current_hash',
                                                        updated = '$entry_timestamp_fmt',
+                                                       $tsvector_qpart
                                                        num_comments = '$num_comments',
                                                        plugin_data = '$entry_plugin_data',
                                                        author = '$entry_author',