[tt-rss.git] / update_daemon2.php

#!/usr/bin/php
<?php
	// This is an experimental multiprocess update daemon.
	// Some configurable variable may be found below.

	// define('DEFAULT_ERROR_LEVEL', E_ALL);
	define('DEFAULT_ERROR_LEVEL', E_ERROR | E_WARNING | E_PARSE);

	declare(ticks = 1);

	define('MAGPIE_CACHE_DIR', '/var/tmp/magpie-ttrss-cache-daemon');
	define('SIMPLEPIE_CACHE_DIR',	'/var/tmp/simplepie-ttrss-cache-daemon');
	define('DISABLE_SESSIONS', true);

	define('MAX_JOBS', 2);

	require_once "version.php";

	if (strpos(VERSION, ".99") !== false || getenv('DAEMON_XDEBUG')) {
		define('DAEMON_EXTENDED_DEBUG', true);
	}

	define('PURGE_INTERVAL', 3600); // seconds

	require_once "sanity_check.php";
	require_once "config.php";

	define('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL);

	if (!ENABLE_UPDATE_DAEMON) {
		die("Please enable option ENABLE_UPDATE_DAEMON in config.php\n");
	}
	
	require_once "db.php";
	require_once "db-prefs.php";
	require_once "functions.php";
	require_once "magpierss/rss_fetch.inc";

	error_reporting(DEFAULT_ERROR_LEVEL);

	$running_jobs = 0;
	$last_checkpoint = -1;

	function sigalrm_handler() {
		die("received SIGALRM, hang in feed update?\n");
	}

	function sigchld_handler($signal) {
		global $running_jobs;
		if ($running_jobs > 0) $running_jobs--;
		print posix_getpid() . ": SIGCHLD received, jobs left: $running_jobs\n";
		pcntl_waitpid(-1, $status, WNOHANG);
	}

	function sigint_handler() {
		unlink(LOCK_DIRECTORY . "/update_daemon.lock");
		die("Received SIGINT. Exiting.\n");
	}

	pcntl_signal(SIGALRM, 'sigalrm_handler');
	pcntl_signal(SIGCHLD, 'sigchld_handler');
	pcntl_signal(SIGINT, 'sigint_handler');

	if (file_is_locked("update_daemon.lock")) {
		die("error: Can't create lockfile. ".
			"Maybe another daemon is already running.\n");
	}

	if (file_is_locked("update_daemon.lock")) {
		die("error: Can't create lockfile. ".
			"Maybe another daemon is already running.\n");
	}

	if (!pcntl_fork()) {
		$lock_handle = make_lockfile("update_daemon.lock");

		if (!$lock_handle) {
			die("error: Can't create lockfile. ".
				"Maybe another daemon is already running.\n");
		}

		while (true) { sleep(100); }
	}

	// Testing database connection.
	// It is unnecessary to start the fork loop if database is not ok.
	$link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME);	

	if (!$link) {
		if (DB_TYPE == "mysql") {
			print mysql_error();
		}
		// PG seems to display its own errors just fine by default.		
		return;
	}

	db_close($link);


	while (true) {

		$next_spawn = $last_checkpoint + SPAWN_INTERVAL - time();

		if ($next_spawn % 10 == 0) {
			print "[MASTER] active jobs: $running_jobs, next spawn at $next_spawn sec\n";
		}

		if ($last_checkpoint + SPAWN_INTERVAL < time()) {

			for ($j = $running_jobs; $j < MAX_JOBS; $j++) {
				print "[MASTER] spawning client $j...";
				$pid = pcntl_fork();
				if ($pid == -1) {
					die("fork failed!\n");
				} else if ($pid) {
					$running_jobs++;
					print "OK [$running_jobs]\n";
				} else {
					pcntl_signal(SIGCHLD, SIG_IGN);
					pcntl_signal(SIGINT, SIG_DFL);

					// ****** Updating RSS code *******
					// Only run in fork process.

					$start_timestamp = time();

					$link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME);	

					if (!$link) {
						if (DB_TYPE == "mysql") {
							print mysql_error();
						}
						// PG seems to display its own errors just fine by default.		
						return;
					}

					if (DB_TYPE == "pgsql") {
						pg_query("set client_encoding = 'utf-8'");
						pg_set_client_encoding("UNICODE");
					} else {
						if (defined('MYSQL_CHARSET') && MYSQL_CHARSET) {
							db_query($link, "SET NAMES " . MYSQL_CHARSET);
							// db_query($link, "SET CHARACTER SET " . MYSQL_CHARSET);
						}
					}

					// We disable stamp file, since it is of no use in a multiprocess update.
					// not really, tho for the time being -fox
					if (!make_stampfile('update_daemon.stamp')) {
						print "warning: unable to create stampfile";
					}	

					// $last_purge = 0;

					// if (time() - $last_purge > PURGE_INTERVAL) {

					// FIXME : $last_purge is of no use in a multiprocess update.
					// FIXME : We ALWAYS purge old posts.
					_debug("Purging old posts (random 30 feeds)...");
					global_purge_old_posts($link, true, 30);

					// 	$last_purge = time();
					// }

					// Process all other feeds using last_updated and interval parameters

					$random_qpart = sql_random_function();
						
					if (DAEMON_UPDATE_LOGIN_LIMIT > 0) {
						if (DB_TYPE == "pgsql") {
							$login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'";
						} else {
							$login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)";
						}			
					} else {
						$login_thresh_qpart = "";
					}

					if (DB_TYPE == "pgsql") {
						$update_limit_qpart = "AND ttrss_feeds.last_updated < NOW() - INTERVAL '".(DAEMON_SLEEP_INTERVAL*2)." seconds'";
					} else {
						$update_limit_qpart = "AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ".(DAEMON_SLEEP_INTERVAL*2)." SECOND)";
					}

					if (DB_TYPE == "pgsql") {
							$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '120 seconds')";
						} else {
							$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 120 SECOND))";
						}			

					$result = db_query($link, "SELECT feed_url,ttrss_feeds.id,owner_uid,
							SUBSTRING(last_updated,1,19) AS last_updated,
							update_interval 
						FROM 
							ttrss_feeds,ttrss_users 
						WHERE 
							ttrss_users.id = owner_uid $login_thresh_qpart $update_limit_qpart 
							$updstart_thresh_qpart
						ORDER BY $random_qpart DESC LIMIT " . DAEMON_FEED_LIMIT);

					$user_prefs_cache = array();

					_debug(sprintf("Scheduled %d feeds to update...\n", db_num_rows($result)));

					// Here is a little cache magic in order to minimize risk of double feed updates.
					$feeds_to_update = array();
					while ($line = db_fetch_assoc($result)) {
						$feeds_to_update[$line['id']] = $line;
					}

					// We update the feed last update started date before anything else.
					// There is no lag due to feed contents downloads
					// It prevent an other process to update the same feed.
					$feed_ids = array_keys($feeds_to_update);
					if($feed_ids) {
						db_query($link, sprintf("UPDATE ttrss_feeds SET last_update_started = NOW()
							WHERE id IN (%s)", implode(',', $feed_ids)));
					}

					while ($line = array_pop($feeds_to_update)) {

						$upd_intl = $line["update_interval"];
						$user_id = $line["owner_uid"];

						if (!$upd_intl || $upd_intl == 0) {
							if (!$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL']) {			
								$upd_intl = get_pref($link, 'DEFAULT_UPDATE_INTERVAL', $user_id);
								$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'] = $upd_intl;
							} else {
								$upd_intl = $user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'];
							}
						}

						if ($upd_intl < 0) { 
				#				print "Updates disabled.\n";
							continue; 
						}

						_debug("Feed: " . $line["feed_url"] . ", " . $line["last_updated"]);

				//			_debug(sprintf("\tLU: %d, INTL: %d, UID: %d) ", 
				//				time() - strtotime($line["last_updated"]), $upd_intl*60, $user_id));

						if (!$line["last_updated"] || 
							time() - strtotime($line["last_updated"]) > ($upd_intl * 60)) {

							_debug("Updating...");

							pcntl_alarm(300);

							update_rss_feed($link, $line["feed_url"], $line["id"], true);	

							pcntl_alarm(0);

							sleep(1); // prevent flood (FIXME make this an option?)
						} else {
							_debug("Update not needed.");
						}
					}

					if (DAEMON_SENDS_DIGESTS) send_headlines_digests($link);

					print "Elapsed time: " . (time() - $start_timestamp) . " second(s)\n";

					db_close($link);

					// We are in a fork.
					// We wait a little before exiting to avoid to be faster than our parent process.
					sleep(1);
					// We exit in order to avoid fork bombing.
					exit(0);
				}
			}
			$last_checkpoint = time();
		}
		sleep(1);
	}

?>
Commit	Line	Data
02008cb1 AD	1	#!/usr/bin/php
02008cb1 AD	2	<?php
ffa7cbae AD	3	// This is an experimental multiprocess update daemon.
	4	// Some configurable variable may be found below.
	5
	6	// define('DEFAULT_ERROR_LEVEL', E_ALL);
	7	define('DEFAULT_ERROR_LEVEL', E_ERROR \| E_WARNING \| E_PARSE);
02008cb1 AD	8
	9	declare(ticks = 1);
	10
ffa7cbae AD	11	define('MAGPIE_CACHE_DIR', '/var/tmp/magpie-ttrss-cache-daemon');
	12	define('SIMPLEPIE_CACHE_DIR', '/var/tmp/simplepie-ttrss-cache-daemon');
	13	define('DISABLE_SESSIONS', true);
02008cb1 AD	14
02008cb1 AD	15	define('MAX_JOBS', 2);
ffa7cbae AD	16
	17	require_once "version.php";
	18
010c16f1	19	if (strpos(VERSION, ".99") !== false \|\| getenv('DAEMON_XDEBUG')) {
ffa7cbae AD	20	define('DAEMON_EXTENDED_DEBUG', true);
	21	}
	22
	23	define('PURGE_INTERVAL', 3600); // seconds
	24
	25	require_once "sanity_check.php";
	26	require_once "config.php";
	27
02008cb1	28	define('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL);
ffa7cbae AD	29
	30	if (!ENABLE_UPDATE_DAEMON) {
	31	die("Please enable option ENABLE_UPDATE_DAEMON in config.php\n");
	32	}
	33
	34	require_once "db.php";
	35	require_once "db-prefs.php";
	36	require_once "functions.php";
	37	require_once "magpierss/rss_fetch.inc";
	38
	39	error_reporting(DEFAULT_ERROR_LEVEL);
02008cb1 AD	40
	41	$running_jobs = 0;
	42	$last_checkpoint = -1;
	43
ffa7cbae AD	44	function sigalrm_handler() {
	45	die("received SIGALRM, hang in feed update?\n");
	46	}
	47
02008cb1 AD	48	function sigchld_handler($signal) {
	49	global $running_jobs;
	50	if ($running_jobs > 0) $running_jobs--;
	51	print posix_getpid() . ": SIGCHLD received, jobs left: $running_jobs\n";
	52	pcntl_waitpid(-1, $status, WNOHANG);
	53	}
	54
6a69e61f AD	55	function sigint_handler() {
	56	unlink(LOCK_DIRECTORY . "/update_daemon.lock");
	57	die("Received SIGINT. Exiting.\n");
	58	}
	59
ffa7cbae	60	pcntl_signal(SIGALRM, 'sigalrm_handler');
02008cb1	61	pcntl_signal(SIGCHLD, 'sigchld_handler');
6a69e61f AD	62	pcntl_signal(SIGINT, 'sigint_handler');
6a69e61f AD	63
884c0a36 AD	64	if (file_is_locked("update_daemon.lock")) {
884c0a36 AD	65	die("error: Can't create lockfile. ".
6a69e61f AD	66	"Maybe another daemon is already running.\n");
6a69e61f AD	67	}
02008cb1	68
ffa7cbae AD	69	if (file_is_locked("update_daemon.lock")) {
	70	die("error: Can't create lockfile. ".
	71	"Maybe another daemon is already running.\n");
	72	}
	73
884c0a36 AD	74	if (!pcntl_fork()) {
	75	$lock_handle = make_lockfile("update_daemon.lock");
	76
	77	if (!$lock_handle) {
	78	die("error: Can't create lockfile. ".
	79	"Maybe another daemon is already running.\n");
	80	}
	81
	82	while (true) { sleep(100); }
	83	}
	84
ffa7cbae AD	85	// Testing database connection.
	86	// It is unnecessary to start the fork loop if database is not ok.
	87	$link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME);
	88
	89	if (!$link) {
	90	if (DB_TYPE == "mysql") {
	91	print mysql_error();
	92	}
	93	// PG seems to display its own errors just fine by default.
	94	return;
	95	}
	96
	97	db_close($link);
	98
	99
02008cb1 AD	100	while (true) {
	101
	102	$next_spawn = $last_checkpoint + SPAWN_INTERVAL - time();
	103
ce1aa9b7 AD	104	if ($next_spawn % 10 == 0) {
	105	print "[MASTER] active jobs: $running_jobs, next spawn at $next_spawn sec\n";
	106	}
02008cb1 AD	107
	108	if ($last_checkpoint + SPAWN_INTERVAL < time()) {
	109
	110	for ($j = $running_jobs; $j < MAX_JOBS; $j++) {
	111	print "[MASTER] spawning client $j...";
	112	$pid = pcntl_fork();
	113	if ($pid == -1) {
	114	die("fork failed!\n");
	115	} else if ($pid) {
	116	$running_jobs++;
	117	print "OK [$running_jobs]\n";
	118	} else {
	119	pcntl_signal(SIGCHLD, SIG_IGN);
6a69e61f	120	pcntl_signal(SIGINT, SIG_DFL);
ffa7cbae AD	121
	122	// **** Updating RSS code *****
	123	// Only run in fork process.
	124
	125	$start_timestamp = time();
	126
	127	$link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME);
	128
	129	if (!$link) {
	130	if (DB_TYPE == "mysql") {
	131	print mysql_error();
	132	}
	133	// PG seems to display its own errors just fine by default.
	134	return;
	135	}
	136
	137	if (DB_TYPE == "pgsql") {
	138	pg_query("set client_encoding = 'utf-8'");
	139	pg_set_client_encoding("UNICODE");
	140	} else {
	141	if (defined('MYSQL_CHARSET') && MYSQL_CHARSET) {
	142	db_query($link, "SET NAMES " . MYSQL_CHARSET);
	143	// db_query($link, "SET CHARACTER SET " . MYSQL_CHARSET);
	144	}
	145	}
	146
	147	// We disable stamp file, since it is of no use in a multiprocess update.
	148	// not really, tho for the time being -fox
	149	if (!make_stampfile('update_daemon.stamp')) {
	150	print "warning: unable to create stampfile";
	151	}
	152
	153	// $last_purge = 0;
	154
	155	// if (time() - $last_purge > PURGE_INTERVAL) {
	156
	157	// FIXME : $last_purge is of no use in a multiprocess update.
	158	// FIXME : We ALWAYS purge old posts.
	159	_debug("Purging old posts (random 30 feeds)...");
	160	global_purge_old_posts($link, true, 30);
	161
	162	// $last_purge = time();
	163	// }
	164
	165	// Process all other feeds using last_updated and interval parameters
	166
	167	$random_qpart = sql_random_function();
	168
	169	if (DAEMON_UPDATE_LOGIN_LIMIT > 0) {
	170	if (DB_TYPE == "pgsql") {
	171	$login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'";
	172	} else {
	173	$login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)";
	174	}
	175	} else {
	176	$login_thresh_qpart = "";
	177	}
	178
	179	if (DB_TYPE == "pgsql") {
	180	$update_limit_qpart = "AND ttrss_feeds.last_updated < NOW() - INTERVAL '".(DAEMON_SLEEP_INTERVAL*2)." seconds'";
	181	} else {
	182	$update_limit_qpart = "AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ".(DAEMON_SLEEP_INTERVAL*2)." SECOND)";
	183	}
	184
185	if (DB_TYPE == "pgsql") {
186	$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '120 seconds')";
187	} else {
188	$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 120 SECOND))";
189	}
190
191	$result = db_query($link, "SELECT feed_url,ttrss_feeds.id,owner_uid,
192	SUBSTRING(last_updated,1,19) AS last_updated,
193	update_interval
194	FROM
195	ttrss_feeds,ttrss_users
196	WHERE
197	ttrss_users.id = owner_uid $login_thresh_qpart $update_limit_qpart
198	$updstart_thresh_qpart
199	ORDER BY $random_qpart DESC LIMIT " . DAEMON_FEED_LIMIT);
200
201	$user_prefs_cache = array();
202
203	_debug(sprintf("Scheduled %d feeds to update...\n", db_num_rows($result)));
204
205	// Here is a little cache magic in order to minimize risk of double feed updates.
206	$feeds_to_update = array();
207	while ($line = db_fetch_assoc($result)) {
208	$feeds_to_update[$line['id']] = $line;
209	}
210
211	// We update the feed last update started date before anything else.
212	// There is no lag due to feed contents downloads
213	// It prevent an other process to update the same feed.
214	$feed_ids = array_keys($feeds_to_update);
215	if($feed_ids) {
216	db_query($link, sprintf("UPDATE ttrss_feeds SET last_update_started = NOW()
217	WHERE id IN (%s)", implode(',', $feed_ids)));
218	}
219
220	while ($line = array_pop($feeds_to_update)) {
221
222	$upd_intl = $line["update_interval"];
223	$user_id = $line["owner_uid"];
224
225	if (!$upd_intl \|\| $upd_intl == 0) {
226	if (!$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL']) {
227	$upd_intl = get_pref($link, 'DEFAULT_UPDATE_INTERVAL', $user_id);
228	$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'] = $upd_intl;
229	} else {
230	$upd_intl = $user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'];
231	}
232	}
233
234	if ($upd_intl < 0) {
235	# print "Updates disabled.\n";
236	continue;
237	}
238
239	_debug("Feed: " . $line["feed_url"] . ", " . $line["last_updated"]);
240
241	// _debug(sprintf("\tLU: %d, INTL: %d, UID: %d) ",
242	// time() - strtotime($line["last_updated"]), $upd_intl*60, $user_id));
243
244	if (!$line["last_updated"] \|\|
245	time() - strtotime($line["last_updated"]) > ($upd_intl * 60)) {
246
247	_debug("Updating...");
248
249	pcntl_alarm(300);
250
251	update_rss_feed($link, $line["feed_url"], $line["id"], true);
252
253	pcntl_alarm(0);
254
255	sleep(1); // prevent flood (FIXME make this an option?)
256	} else {
257	_debug("Update not needed.");
258	}
259	}
260
261	if (DAEMON_SENDS_DIGESTS) send_headlines_digests($link);
262
263	print "Elapsed time: " . (time() - $start_timestamp) . " second(s)\n";
264
265	db_close($link);
266
267	// We are in a fork.
268	// We wait a little before exiting to avoid to be faster than our parent process.
269	sleep(1);
270	// We exit in order to avoid fork bombing.
02008cb1 AD	271	exit(0);
	272	}
	273	}
	274	$last_checkpoint = time();
	275	}
	276	sleep(1);
	277	}
	278
	279	?>