<commit_before><?php

use Symfony\Component\Finder\Finder;
use eland\util\queue_container;
use eland\util\task_container;

if (php_sapi_name() !== 'cli')
{
	echo '-- cli only --';
	exit;
}

require_once __DIR__ . '/include/worker.php';

$boot = $app['eland.cache']->get('boot');

if (!count($boot))
{
	$boot = ['count' => 0];
}

$boot['count']++;
$app['eland.cache']->set('boot', $boot);

echo 'worker started .. ' . $boot['count'] . "\n";

$queue = new queue_container($app, 'queue');
$task = new task_container($app, 'task');
$schema_task = new task_container($app, 'schema_task');

$loop_count = 1;

$app['redis']->set('block_task', '1');
$app['redis']->expire('block_task', 3);

while (true)
{
	$app['eland.log_db']->update();

	sleep(1);

	if ($queue->should_run())
	{
		$queue->run();
	}
	else if ($task->should_run())
	{
		$task->run();
	}
	else if ($schema_task->should_run())
	{
		$schema_task->run();
	}

	if ($loop_count % 1000 == 0)
	{
		error_log('..worker.. ' . $boot['count'] . ' .. ' . $loop_count);
	}

	$loop_count++;
}
<commit_msg>kill old worker<commit_after><?php

use Symfony\Component\Finder\Finder;
use eland\util\queue_container;
use eland\util\task_container;

if (php_sapi_name() !== 'cli')
{
	echo '-- cli only --';
	exit;
}

require_once __DIR__ . '/include/worker.php';

$boot = $app['eland.cache']->get('boot');

if (!count($boot))
{
	$boot = ['count' => 0];
}

$boot['count']++;
$app['eland.cache']->set('boot', $boot);

echo 'worker started .. ' . $boot['count'] . "\n";

$queue = new queue_container($app, 'queue');
$task = new task_container($app, 'task');
$schema_task = new task_container($app, 'schema_task');

$loop_count = 1;

$app['redis']->set('block_task', '1');
$app['redis']->expire('block_task', 3);

while (true)
{
	$app['eland.log_db']->update();

	sleep(1);

	if ($queue->should_run())
	{
		$queue->run();
	}
	else if ($task->should_run())
	{
		$task->run();
	}
	else if ($schema_task->should_run())
	{
		$schema_task->run();
	}

	if ($loop_count % 1000 === 0)
	{
		error_log('..worker.. ' . $boot['count'] . ' .. ' . $loop_count);
	}

	if ($loop_count % 5 === 0)
	{
		$boot_test = $app['cache']->get('boot');

		if ($boot_test['count'] < $boot['count'])
		{
			exit;
		}
	}

	$loop_count++;
}

{'fraction_non_alphanumeric': 0.2550156041016496, 'fraction_numerical': 0.00980829246544806, 'mean_word_length': 4.242990654205608, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6012849', 'n_tokens_mistral': 950, 'n_tokens_neox': 913, 'n_words': 158}