Tiny Tiny RSS data update script.

X-Git-Url: https://git.wh0rd.org/?a=blobdiff_plain;f=update.php;h=17eafe126675359e120034f52a7d7e0e19d75078;hb=e2b8c9273e09091c235959c25d8e4d8122aa6ca8;hp=821d25bce27c5262d65cdfddd62adea65b88e24c;hpb=17a8e61d2ae9e938aaf60292666b6ccf5cb09067;p=tt-rss.git diff --git a/update.php b/update.php index 821d25bc..17eafe12 100755 --- a/update.php +++ b/update.php @@ -9,7 +9,6 @@ require_once "autoload.php"; require_once "functions.php"; - require_once "rssfuncs.php"; require_once "config.php"; require_once "sanity_check.php"; require_once "db.php"; @@ -18,6 +17,8 @@ if (!defined('PHP_EXECUTABLE')) define('PHP_EXECUTABLE', '/usr/bin/php'); + $pdo = Db::pdo(); + init_plugins(); $longopts = array("feeds", @@ -28,6 +29,7 @@ "cleanup-tags", "quiet", "log:", + "log-level:", "indexes", "pidlock:", "update-schema", @@ -38,7 +40,6 @@ "debug-feed:", "force-refetch", "force-rehash", - "decrypt-feeds", "help"); foreach (PluginHost::getInstance()->get_commands() as $command => $data) { @@ -59,14 +60,13 @@ Tiny Tiny RSS data update script. -

- + query( "UPDATE ttrss_feeds SET + last_update_started = '1970-01-01', last_updated = '1970-01-01'"); } if (isset($options["feeds"])) { - update_daemon_common(); - housekeeping_common(true); + RSSUtils::update_daemon_common(); + RSSUtils::housekeeping_common(true); PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK, "hook_update_task", $op); } if (isset($options["feedbrowser"])) { - $count = update_feedbrowser_cache(); + $count = RSSUtils::update_feedbrowser_cache(); print "Finished, $count feeds processed.\n"; } if (isset($options["daemon"])) { while (true) { $quiet = (isset($options["quiet"])) ? "--quiet" : ""; - $log = isset($options['log']) ? '--log '.$options['log'] : ''; + $log = isset($options['log']) ? '--log '.$options['log'] : ''; + $log_level = isset($options['log-level']) ? '--log-level '.$options['log-level'] : ''; - passthru(PHP_EXECUTABLE . " " . $argv[0] ." --daemon-loop $quiet $log"); - _debug("Sleeping for " . DAEMON_SLEEP_INTERVAL . " seconds..."); - sleep(DAEMON_SLEEP_INTERVAL); + passthru(PHP_EXECUTABLE . " " . $argv[0] ." --daemon-loop $quiet $log $log_level"); + + // let's enforce a minimum spawn interval as to not forkbomb the host + $spawn_interval = max(60, DAEMON_SLEEP_INTERVAL); + + Debug::log("Sleeping for $spawn_interval seconds..."); + sleep($spawn_interval); } } if (isset($options["daemon-loop"])) { if (!make_stampfile('update_daemon.stamp')) { - _debug("warning: unable to create stampfile\n"); + Debug::log("warning: unable to create stampfile\n"); } - update_daemon_common(isset($options["pidlock"]) ? 50 : DAEMON_FEED_LIMIT); + RSSUtils::update_daemon_common(isset($options["pidlock"]) ? 50 : DAEMON_FEED_LIMIT); if (!isset($options["pidlock"]) || $options["task"] == 0) - housekeeping_common(true); + RSSUtils::housekeeping_common(true); PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK, "hook_update_task", $op); } if (isset($options["cleanup-tags"])) { $rc = cleanup_tags( 14, 50000); - _debug("$rc tags deleted.\n"); + Debug::log("$rc tags deleted.\n"); } if (isset($options["indexes"])) { - _debug("PLEASE BACKUP YOUR DATABASE BEFORE PROCEEDING!"); - _debug("Type 'yes' to continue."); + Debug::log("PLEASE BACKUP YOUR DATABASE BEFORE PROCEEDING!"); + Debug::log("Type 'yes' to continue."); if (read_stdin() != 'yes') exit; - _debug("clearing existing indexes..."); + Debug::log("clearing existing indexes..."); if (DB_TYPE == "pgsql") { - $result = db_query( "SELECT relname FROM + $sth = $pdo->query( "SELECT relname FROM pg_catalog.pg_class WHERE relname LIKE 'ttrss_%' AND relname NOT LIKE '%_pkey' AND relkind = 'i'"); } else { - $result = db_query( "SELECT index_name,table_name FROM + $sth = $pdo->query( "SELECT index_name,table_name FROM information_schema.statistics WHERE index_name LIKE 'ttrss_%'"); } - while ($line = db_fetch_assoc($result)) { + while ($line = $sth->fetch()) { if (DB_TYPE == "pgsql") { $statement = "DROP INDEX " . $line["relname"]; - _debug($statement); + Debug::log($statement); } else { $statement = "ALTER TABLE ". $line['table_name']." DROP INDEX ".$line['index_name']; - _debug($statement); + Debug::log($statement); } - db_query( $statement, false); + $pdo->query($statement); } - _debug("reading indexes from schema for: " . DB_TYPE); + Debug::log("reading indexes from schema for: " . DB_TYPE); $fp = fopen("schema/ttrss_schema_" . DB_TYPE . ".sql", "r"); if ($fp) { @@ -249,31 +259,31 @@ $statement = "CREATE INDEX $index ON $table"; - _debug($statement); - db_query( $statement); + Debug::log($statement); + $pdo->query($statement); } } fclose($fp); } else { - _debug("unable to open schema file."); + Debug::log("unable to open schema file."); } - _debug("all done."); + Debug::log("all done."); } if (isset($options["convert-filters"])) { - _debug("WARNING: this will remove all existing type2 filters."); - _debug("Type 'yes' to continue."); + Debug::log("WARNING: this will remove all existing type2 filters."); + Debug::log("Type 'yes' to continue."); if (read_stdin() != 'yes') exit; - _debug("converting filters..."); + Debug::log("converting filters..."); - db_query( "DELETE FROM ttrss_filters2"); + $pdo->query("DELETE FROM ttrss_filters2"); - $result = db_query( "SELECT * FROM ttrss_filters ORDER BY id"); + $res = $pdo->query("SELECT * FROM ttrss_filters ORDER BY id"); - while ($line = db_fetch_assoc($result)) { + while ($line = $res->fetch()) { $owner_uid = $line["owner_uid"]; // date filters are removed @@ -312,30 +322,30 @@ } if (isset($options["update-schema"])) { - _debug("checking for updates (" . DB_TYPE . ")..."); + Debug::log("checking for updates (" . DB_TYPE . ")..."); - $updater = new DbUpdater(Db::get(), DB_TYPE, SCHEMA_VERSION); + $updater = new DbUpdater(Db::pdo(), DB_TYPE, SCHEMA_VERSION); if ($updater->isUpdateRequired()) { - _debug("schema update required, version " . $updater->getSchemaVersion() . " to " . SCHEMA_VERSION); - _debug("WARNING: please backup your database before continuing."); - _debug("Type 'yes' to continue."); + Debug::log("schema update required, version " . $updater->getSchemaVersion() . " to " . SCHEMA_VERSION); + Debug::log("WARNING: please backup your database before continuing."); + Debug::log("Type 'yes' to continue."); if (read_stdin() != 'yes') exit; for ($i = $updater->getSchemaVersion() + 1; $i <= SCHEMA_VERSION; $i++) { - _debug("performing update up to version $i..."); + Debug::log("performing update up to version $i..."); $result = $updater->performUpdateTo($i, false); - _debug($result ? "OK!" : "FAILED!"); + Debug::log($result ? "OK!" : "FAILED!"); if (!$result) return; } } else { - _debug("update not required."); + Debug::log("update not required."); } } @@ -343,28 +353,35 @@ if (isset($options["gen-search-idx"])) { echo "Generating search index (stemming set to English)...\n"; - $result = db_query("SELECT COUNT(id) AS count FROM ttrss_entries WHERE tsvector_combined IS NULL"); - $count = db_fetch_result($result, 0, "count"); + $res = $pdo->query("SELECT COUNT(id) AS count FROM ttrss_entries WHERE tsvector_combined IS NULL"); + $row = $res->fetch(); + $count = $row['count']; print "Articles to process: $count.\n"; $limit = 500; $processed = 0; + $sth = $pdo->prepare("SELECT id, title, content FROM ttrss_entries WHERE + tsvector_combined IS NULL ORDER BY id LIMIT ?"); + $sth->execute([$limit]); + + $usth = $pdo->prepare("UPDATE ttrss_entries + SET tsvector_combined = to_tsvector('english', ?) WHERE id = ?"); + while (true) { - $result = db_query("SELECT id, title, content FROM ttrss_entries WHERE tsvector_combined IS NULL ORDER BY id LIMIT $limit"); - while ($line = db_fetch_assoc($result)) { - $tsvector_combined = db_escape_string(mb_substr($line['title'] . ' ' . strip_tags(str_replace('<', ' <', $line['content'])), - 0, 1000000)); + while ($line = $sth->fetch()) { + $tsvector_combined = mb_substr(strip_tags($line["title"] . " " . $line["content"]), 0, 1000000); + + $usth->execute([$tsvector_combined, $line['id']]); - db_query("UPDATE ttrss_entries SET tsvector_combined = to_tsvector('english', '$tsvector_combined') WHERE id = " . $line["id"]); + $processed++; } - $processed += db_num_rows($result); print "Processed $processed articles...\n"; - if (db_num_rows($result) != $limit) { + if ($processed < $limit) { echo "All done.\n"; break; } @@ -399,39 +416,11 @@ if (isset($options["force-refetch"])) $_REQUEST["force_refetch"] = true; if (isset($options["force-rehash"])) $_REQUEST["force_rehash"] = true; - $_REQUEST['xdebug'] = 1; - - update_rss_feed($feed); - } - - if (isset($options["decrypt-feeds"])) { - $result = db_query("SELECT id, auth_pass FROM ttrss_feeds WHERE auth_pass_encrypted = true"); - - if (!function_exists("mcrypt_decrypt")) { - _debug("mcrypt functions not available."); - return; - } - - require_once "crypt.php"; - - $total = 0; - - db_query("BEGIN"); - - while ($line = db_fetch_assoc($result)) { - _debug("processing feed id " . $line["id"]); - - $auth_pass = db_escape_string(decrypt_string($line["auth_pass"])); - - db_query("UPDATE ttrss_feeds SET auth_pass_encrypted = false, auth_pass = '$auth_pass' - WHERE id = " . $line["id"]); - - ++$total; - } + Debug::set_loglevel(Debug::$LOG_EXTENDED); - db_query("COMMIT"); + $rc = RSSUtils::update_rss_feed($feed) != false ? 0 : 1; - _debug("$total feeds processed."); + exit($rc); } PluginHost::getInstance()->run_commands($options);