[tt-rss.git] / update_daemon2.php

#!/usr/bin/php
<?php
	// This is an experimental multiprocess update daemon.
	// Some configurable variable may be found below.

	// define('DEFAULT_ERROR_LEVEL', E_ALL);
	define('DEFAULT_ERROR_LEVEL', E_ERROR | E_WARNING | E_PARSE);

	declare(ticks = 1);

	define('MAGPIE_CACHE_DIR', '/var/tmp/magpie-ttrss-cache-daemon');
	define('SIMPLEPIE_CACHE_DIR',	'/var/tmp/simplepie-ttrss-cache-daemon');
	define('DISABLE_SESSIONS', true);

	define('MAX_JOBS', 2);

	require_once "version.php";

	if (strpos(VERSION, ".99") !== false || getenv('DAEMON_XDEBUG')) {
		define('DAEMON_EXTENDED_DEBUG', true);
	}

	define('PURGE_INTERVAL', 3600); // seconds

	require_once "sanity_check.php";
	require_once "config.php";

	define('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL);

	if (!ENABLE_UPDATE_DAEMON) {
		die("Please enable option ENABLE_UPDATE_DAEMON in config.php\n");
	}
	
	require_once "db.php";
	require_once "db-prefs.php";
	require_once "functions.php";
	require_once "magpierss/rss_fetch.inc";

	error_reporting(DEFAULT_ERROR_LEVEL);

	$children = array();

	$last_checkpoint = -1;

	function sigalrm_handler() {
		die("received SIGALRM, hang in feed update?\n");
	}

	function sigchld_handler($signal) {
		global $children;

		$tmp = array();

		foreach ($children as $pid) {
			if (pcntl_waitpid($pid, $status, WNOHANG) != $pid) {
				array_push($tmp, $pid);
			} else {
				_debug("[SIGCHLD] child $pid reaped.");
			}
		}

		$children = $tmp;

		$running_jobs = count($children);

		_debug("[SIGCHLD] jobs left: $running_jobs");
		pcntl_waitpid(-1, $status, WNOHANG);
	}

	function sigint_handler() {
		unlink(LOCK_DIRECTORY . "/update_daemon.lock");
		die("Received SIGINT. Exiting.\n");
	}

	pcntl_signal(SIGALRM, 'sigalrm_handler');
	pcntl_signal(SIGCHLD, 'sigchld_handler');
	pcntl_signal(SIGINT, 'sigint_handler');

	if (file_is_locked("update_daemon.lock")) {
		die("error: Can't create lockfile. ".
			"Maybe another daemon is already running.\n");
	}

	if (file_is_locked("update_daemon.lock")) {
		die("error: Can't create lockfile. ".
			"Maybe another daemon is already running.\n");
	}

	if (!pcntl_fork()) {
		$lock_handle = make_lockfile("update_daemon.lock");

		if (!$lock_handle) {
			die("error: Can't create lockfile. ".
				"Maybe another daemon is already running.\n");
		}

		while (true) { sleep(100); }
	}

	// Testing database connection.
	// It is unnecessary to start the fork loop if database is not ok.
	$link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME);	

	if (!$link) {
		if (DB_TYPE == "mysql") {
			print mysql_error();
		}
		// PG seems to display its own errors just fine by default.		
		return;
	}

	db_close($link);


	while (true) {

		$next_spawn = $last_checkpoint + SPAWN_INTERVAL - time();

		if ($next_spawn % 10 == 0) {
			$running_jobs = count($children);
			_debug("[MASTER] active jobs: $running_jobs, next spawn at $next_spawn sec.");
		}

		if ($last_checkpoint + SPAWN_INTERVAL < time()) {

			for ($j = count($children); $j < MAX_JOBS; $j++) {
				$pid = pcntl_fork();
				if ($pid == -1) {
					die("fork failed!\n");
				} else if ($pid) {
					_debug("[MASTER] spawned client $j [PID:$pid]...");
					array_push($children, $pid);
				} else {
					pcntl_signal(SIGCHLD, SIG_IGN);
					pcntl_signal(SIGINT, SIG_DFL);

					// ****** Updating RSS code *******
					// Only run in fork process.

					$start_timestamp = time();

					$link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME);	

					if (!$link) {
						if (DB_TYPE == "mysql") {
							print mysql_error();
						}
						// PG seems to display its own errors just fine by default.		
						return;
					}

					if (DB_TYPE == "pgsql") {
						pg_query("set client_encoding = 'utf-8'");
						pg_set_client_encoding("UNICODE");
					} else {
						if (defined('MYSQL_CHARSET') && MYSQL_CHARSET) {
							db_query($link, "SET NAMES " . MYSQL_CHARSET);
							// db_query($link, "SET CHARACTER SET " . MYSQL_CHARSET);
						}
					}

					// We disable stamp file, since it is of no use in a multiprocess update.
					// not really, tho for the time being -fox
					if (!make_stampfile('update_daemon.stamp')) {
						print "warning: unable to create stampfile";
					}	

					// $last_purge = 0;

					// if (time() - $last_purge > PURGE_INTERVAL) {

					// FIXME : $last_purge is of no use in a multiprocess update.
					// FIXME : We ALWAYS purge old posts.
					_debug("Purging old posts (random 30 feeds)...");
					global_purge_old_posts($link, true, 30);

					// 	$last_purge = time();
					// }

					// Process all other feeds using last_updated and interval parameters

					$random_qpart = sql_random_function();
						
					if (DAEMON_UPDATE_LOGIN_LIMIT > 0) {
						if (DB_TYPE == "pgsql") {
							$login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'";
						} else {
							$login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)";
						}			
					} else {
						$login_thresh_qpart = "";
					}

					if (DB_TYPE == "pgsql") {
						$update_limit_qpart = "AND ttrss_feeds.last_updated < NOW() - INTERVAL '".(DAEMON_SLEEP_INTERVAL*2)." seconds'";
					} else {
						$update_limit_qpart = "AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ".(DAEMON_SLEEP_INTERVAL*2)." SECOND)";
					}

					if (DB_TYPE == "pgsql") {
							$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '120 seconds')";
						} else {
							$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 120 SECOND))";
						}			

					$result = db_query($link, "SELECT feed_url,ttrss_feeds.id,owner_uid,
							SUBSTRING(last_updated,1,19) AS last_updated,
							update_interval 
						FROM 
							ttrss_feeds,ttrss_users 
						WHERE 
							ttrss_users.id = owner_uid $login_thresh_qpart $update_limit_qpart 
							$updstart_thresh_qpart
						ORDER BY $random_qpart DESC LIMIT " . DAEMON_FEED_LIMIT);

					$user_prefs_cache = array();

					_debug(sprintf("Scheduled %d feeds to update...\n", db_num_rows($result)));

					// Here is a little cache magic in order to minimize risk of double feed updates.
					$feeds_to_update = array();
					while ($line = db_fetch_assoc($result)) {
						$feeds_to_update[$line['id']] = $line;
					}

					// We update the feed last update started date before anything else.
					// There is no lag due to feed contents downloads
					// It prevent an other process to update the same feed.
					$feed_ids = array_keys($feeds_to_update);
					if($feed_ids) {
						db_query($link, sprintf("UPDATE ttrss_feeds SET last_update_started = NOW()
							WHERE id IN (%s)", implode(',', $feed_ids)));
					}

					while ($line = array_pop($feeds_to_update)) {

						$upd_intl = $line["update_interval"];
						$user_id = $line["owner_uid"];

						if (!$upd_intl || $upd_intl == 0) {
							if (!$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL']) {			
								$upd_intl = get_pref($link, 'DEFAULT_UPDATE_INTERVAL', $user_id);
								$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'] = $upd_intl;
							} else {
								$upd_intl = $user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'];
							}
						}

						if ($upd_intl < 0) { 
				#				print "Updates disabled.\n";
							continue; 
						}

						_debug("Feed: " . $line["feed_url"] . ", " . $line["last_updated"]);

				//			_debug(sprintf("\tLU: %d, INTL: %d, UID: %d) ", 
				//				time() - strtotime($line["last_updated"]), $upd_intl*60, $user_id));

						if (!$line["last_updated"] || 
							time() - strtotime($line["last_updated"]) > ($upd_intl * 60)) {

							_debug("Updating...");

							pcntl_alarm(300);

							update_rss_feed($link, $line["feed_url"], $line["id"], true);	

							pcntl_alarm(0);

							sleep(1); // prevent flood (FIXME make this an option?)
						} else {
							_debug("Update not needed.");
						}
					}

					if (DAEMON_SENDS_DIGESTS) send_headlines_digests($link);

					print "Elapsed time: " . (time() - $start_timestamp) . " second(s)\n";

					db_close($link);

					// We are in a fork.
					// We wait a little before exiting to avoid to be faster than our parent process.
					sleep(1);
					// We exit in order to avoid fork bombing.
					exit(0);
				}
			}
			$last_checkpoint = time();
		}
		sleep(1);
	}

?>
Commit	Line	Data
02008cb1 AD	1	#!/usr/bin/php
02008cb1 AD	2	<?php
ffa7cbae AD	3	// This is an experimental multiprocess update daemon.
	4	// Some configurable variable may be found below.
	5
	6	// define('DEFAULT_ERROR_LEVEL', E_ALL);
	7	define('DEFAULT_ERROR_LEVEL', E_ERROR \| E_WARNING \| E_PARSE);
02008cb1 AD	8
	9	declare(ticks = 1);
	10
ffa7cbae AD	11	define('MAGPIE_CACHE_DIR', '/var/tmp/magpie-ttrss-cache-daemon');
	12	define('SIMPLEPIE_CACHE_DIR', '/var/tmp/simplepie-ttrss-cache-daemon');
	13	define('DISABLE_SESSIONS', true);
02008cb1 AD	14
02008cb1 AD	15	define('MAX_JOBS', 2);
ffa7cbae AD	16
	17	require_once "version.php";
	18
010c16f1	19	if (strpos(VERSION, ".99") !== false \|\| getenv('DAEMON_XDEBUG')) {
ffa7cbae AD	20	define('DAEMON_EXTENDED_DEBUG', true);
	21	}
	22
	23	define('PURGE_INTERVAL', 3600); // seconds
	24
	25	require_once "sanity_check.php";
	26	require_once "config.php";
	27
02008cb1	28	define('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL);
ffa7cbae AD	29
	30	if (!ENABLE_UPDATE_DAEMON) {
	31	die("Please enable option ENABLE_UPDATE_DAEMON in config.php\n");
	32	}
	33
	34	require_once "db.php";
	35	require_once "db-prefs.php";
	36	require_once "functions.php";
	37	require_once "magpierss/rss_fetch.inc";
	38
	39	error_reporting(DEFAULT_ERROR_LEVEL);
02008cb1	40
e9338405 AD	41	$children = array();
e9338405 AD	42
02008cb1 AD	43	$last_checkpoint = -1;
02008cb1 AD	44
ffa7cbae AD	45	function sigalrm_handler() {
	46	die("received SIGALRM, hang in feed update?\n");
	47	}
	48
02008cb1	49	function sigchld_handler($signal) {
e9338405 AD	50	global $children;
	51
	52	$tmp = array();
	53
	54	foreach ($children as $pid) {
	55	if (pcntl_waitpid($pid, $status, WNOHANG) != $pid) {
	56	array_push($tmp, $pid);
	57	} else {
	58	_debug("[SIGCHLD] child $pid reaped.");
	59	}
	60	}
	61
	62	$children = $tmp;
	63
	64	$running_jobs = count($children);
	65
	66	_debug("[SIGCHLD] jobs left: $running_jobs");
02008cb1 AD	67	pcntl_waitpid(-1, $status, WNOHANG);
	68	}
	69
6a69e61f AD	70	function sigint_handler() {
	71	unlink(LOCK_DIRECTORY . "/update_daemon.lock");
	72	die("Received SIGINT. Exiting.\n");
	73	}
	74
ffa7cbae	75	pcntl_signal(SIGALRM, 'sigalrm_handler');
02008cb1	76	pcntl_signal(SIGCHLD, 'sigchld_handler');
6a69e61f AD	77	pcntl_signal(SIGINT, 'sigint_handler');
6a69e61f AD	78
884c0a36 AD	79	if (file_is_locked("update_daemon.lock")) {
884c0a36 AD	80	die("error: Can't create lockfile. ".
6a69e61f AD	81	"Maybe another daemon is already running.\n");
6a69e61f AD	82	}
02008cb1	83
ffa7cbae AD	84	if (file_is_locked("update_daemon.lock")) {
	85	die("error: Can't create lockfile. ".
	86	"Maybe another daemon is already running.\n");
	87	}
	88
884c0a36 AD	89	if (!pcntl_fork()) {
	90	$lock_handle = make_lockfile("update_daemon.lock");
	91
	92	if (!$lock_handle) {
	93	die("error: Can't create lockfile. ".
	94	"Maybe another daemon is already running.\n");
	95	}
	96
	97	while (true) { sleep(100); }
	98	}
	99
ffa7cbae AD	100	// Testing database connection.
	101	// It is unnecessary to start the fork loop if database is not ok.
	102	$link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME);
	103
	104	if (!$link) {
	105	if (DB_TYPE == "mysql") {
	106	print mysql_error();
	107	}
	108	// PG seems to display its own errors just fine by default.
	109	return;
	110	}
	111
	112	db_close($link);
	113
	114
02008cb1 AD	115	while (true) {
	116
	117	$next_spawn = $last_checkpoint + SPAWN_INTERVAL - time();
	118
ce1aa9b7	119	if ($next_spawn % 10 == 0) {
e9338405 AD	120	$running_jobs = count($children);
e9338405 AD	121	_debug("[MASTER] active jobs: $running_jobs, next spawn at $next_spawn sec.");
ce1aa9b7	122	}
02008cb1 AD	123
	124	if ($last_checkpoint + SPAWN_INTERVAL < time()) {
	125
e9338405	126	for ($j = count($children); $j < MAX_JOBS; $j++) {
02008cb1 AD	127	$pid = pcntl_fork();
	128	if ($pid == -1) {
	129	die("fork failed!\n");
	130	} else if ($pid) {
e9338405 AD	131	_debug("[MASTER] spawned client $j [PID:$pid]...");
e9338405 AD	132	array_push($children, $pid);
02008cb1 AD	133	} else {
02008cb1 AD	134	pcntl_signal(SIGCHLD, SIG_IGN);
6a69e61f	135	pcntl_signal(SIGINT, SIG_DFL);
ffa7cbae AD	136
	137	// **** Updating RSS code *****
	138	// Only run in fork process.
	139
	140	$start_timestamp = time();
	141
	142	$link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME);
	143
	144	if (!$link) {
	145	if (DB_TYPE == "mysql") {
	146	print mysql_error();
	147	}
	148	// PG seems to display its own errors just fine by default.
	149	return;
	150	}
	151
	152	if (DB_TYPE == "pgsql") {
	153	pg_query("set client_encoding = 'utf-8'");
	154	pg_set_client_encoding("UNICODE");
	155	} else {
	156	if (defined('MYSQL_CHARSET') && MYSQL_CHARSET) {
	157	db_query($link, "SET NAMES " . MYSQL_CHARSET);
	158	// db_query($link, "SET CHARACTER SET " . MYSQL_CHARSET);
	159	}
	160	}
	161
	162	// We disable stamp file, since it is of no use in a multiprocess update.
	163	// not really, tho for the time being -fox
	164	if (!make_stampfile('update_daemon.stamp')) {
	165	print "warning: unable to create stampfile";
	166	}
	167
	168	// $last_purge = 0;
	169
	170	// if (time() - $last_purge > PURGE_INTERVAL) {
	171
	172	// FIXME : $last_purge is of no use in a multiprocess update.
	173	// FIXME : We ALWAYS purge old posts.
	174	_debug("Purging old posts (random 30 feeds)...");
	175	global_purge_old_posts($link, true, 30);
	176
	177	// $last_purge = time();
	178	// }
	179
	180	// Process all other feeds using last_updated and interval parameters
	181
	182	$random_qpart = sql_random_function();
	183
	184	if (DAEMON_UPDATE_LOGIN_LIMIT > 0) {
	185	if (DB_TYPE == "pgsql") {
	186	$login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'";
	187	} else {
	188	$login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)";
	189	}
	190	} else {
	191	$login_thresh_qpart = "";
	192	}
	193
	194	if (DB_TYPE == "pgsql") {
	195	$update_limit_qpart = "AND ttrss_feeds.last_updated < NOW() - INTERVAL '".(DAEMON_SLEEP_INTERVAL*2)." seconds'";
	196	} else {
	197	$update_limit_qpart = "AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ".(DAEMON_SLEEP_INTERVAL*2)." SECOND)";
	198	}
	199
200	if (DB_TYPE == "pgsql") {
201	$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '120 seconds')";
202	} else {
203	$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 120 SECOND))";
204	}
205
206	$result = db_query($link, "SELECT feed_url,ttrss_feeds.id,owner_uid,
207	SUBSTRING(last_updated,1,19) AS last_updated,
208	update_interval
209	FROM
210	ttrss_feeds,ttrss_users
211	WHERE
212	ttrss_users.id = owner_uid $login_thresh_qpart $update_limit_qpart
213	$updstart_thresh_qpart
214	ORDER BY $random_qpart DESC LIMIT " . DAEMON_FEED_LIMIT);
215
216	$user_prefs_cache = array();
217
218	_debug(sprintf("Scheduled %d feeds to update...\n", db_num_rows($result)));
219
220	// Here is a little cache magic in order to minimize risk of double feed updates.
221	$feeds_to_update = array();
222	while ($line = db_fetch_assoc($result)) {
223	$feeds_to_update[$line['id']] = $line;
224	}
225
226	// We update the feed last update started date before anything else.
227	// There is no lag due to feed contents downloads
228	// It prevent an other process to update the same feed.
229	$feed_ids = array_keys($feeds_to_update);
230	if($feed_ids) {
231	db_query($link, sprintf("UPDATE ttrss_feeds SET last_update_started = NOW()
232	WHERE id IN (%s)", implode(',', $feed_ids)));
233	}
234
235	while ($line = array_pop($feeds_to_update)) {
236
237	$upd_intl = $line["update_interval"];
238	$user_id = $line["owner_uid"];
239
240	if (!$upd_intl \|\| $upd_intl == 0) {
241	if (!$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL']) {
242	$upd_intl = get_pref($link, 'DEFAULT_UPDATE_INTERVAL', $user_id);
243	$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'] = $upd_intl;
244	} else {
245	$upd_intl = $user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'];
246	}
247	}
248
249	if ($upd_intl < 0) {
250	# print "Updates disabled.\n";
251	continue;
252	}
253
254	_debug("Feed: " . $line["feed_url"] . ", " . $line["last_updated"]);
255
256	// _debug(sprintf("\tLU: %d, INTL: %d, UID: %d) ",
257	// time() - strtotime($line["last_updated"]), $upd_intl*60, $user_id));
258
259	if (!$line["last_updated"] \|\|
260	time() - strtotime($line["last_updated"]) > ($upd_intl * 60)) {
261
262	_debug("Updating...");
263
264	pcntl_alarm(300);
265
266	update_rss_feed($link, $line["feed_url"], $line["id"], true);
267
268	pcntl_alarm(0);
269
270	sleep(1); // prevent flood (FIXME make this an option?)
271	} else {
272	_debug("Update not needed.");
273	}
274	}
275
276	if (DAEMON_SENDS_DIGESTS) send_headlines_digests($link);
277
278	print "Elapsed time: " . (time() - $start_timestamp) . " second(s)\n";
279
280	db_close($link);
281
282	// We are in a fork.
283	// We wait a little before exiting to avoid to be faster than our parent process.
284	sleep(1);
285	// We exit in order to avoid fork bombing.
02008cb1 AD	286	exit(0);
	287	}
	288	}
	289	$last_checkpoint = time();
	290	}
	291	sleep(1);
	292	}
	293
	294	?>