<commit_before><?php
/**
 * Will schedual execution of external commands
 * Copied base functionality from KScheduler
 *
 * @package Scheduler
 */
class KGenericScheduler
{
	/**
	 * @var bool
	 */
	private $enableDebug = true;
	
	/**
	 * @var KSchedulerConfig
	 */
	private $schedulerConfig = null;
	
	/**
	 * @var bool
	 */
	private $keepRunning = true;
	
	private $logDir = "c:/web/kaltura/log";
	private $phpPath = null;
	private $statusInterval;
	private $schedulerStatusInterval;
	private $nextStatusTime = 0;
	private $nextSchedulerStatusTime = 0;
	
	/**
	 * Stores all groups of tasks, the index is the task type
	 * @var array is an array of array
	 */
	private $runningTasks = array();
	
	/**
	 * Stores the last execution time of each task type
	 * @var array
	 */
	private $lastRunTime = array();
	
	/**
	 * Stores the last executed index of each task type
	 * @var array
	 */
	private $nextRunIndex = array();
	
	/**
	 * Stores the size of the queue on the server of each task type
	 * @var array
	 */
	private $queueSizes = array();
	
	/**
	 * @param string $phpPath
	 * @param string $configFileName
	 */
	public function __construct($phpPath, $configFileName)
	{
		$this->phpPath = $phpPath;
		$this->loadConfig($configFileName);
	}
	
	public function __destruct()
	{
		$this->_cleanup();
	}
	
	private function initAllWorkers()
	{
		$taskConfigs = $this->schedulerConfig->getTaskConfigList();
		
		foreach($taskConfigs as $taskConfig)
		{
			if(!$taskConfig->type)
				continue;
			
			$tmpConfig = clone $taskConfig;
			$tmpConfig->setInitOnly(true);
			
			KalturaLog::info('Initilizing ' . $tmpConfig->name);
			$tasksetPath = $this->schedulerConfig->getTasksetPath();
			$proc = new KProcessWrapper(0, $this->logDir, $this->phpPath, $tasksetPath, $tmpConfig);
			sleep(1);
		}
	}
	
	private function loadConfig($configFileName = null)
	{
		$firstLoad = is_null($this->schedulerConfig);
		if($firstLoad)
		{
			$this->schedulerConfig = new KSchedulerConfig($configFileName);
			date_default_timezone_set($this->schedulerConfig->getTimezone());
			KalturaLog::info(file_get_contents('VERSION.txt'));
		}
		else
		{	
			if(!$this->schedulerConfig->reloadRequired())
				return;
				
			sleep(2); // make sure the file finsied to be written
			$this->schedulerConfig->load();
		}
		
		KScheduleHelperManager::clearFilters();
		$this->queueSizes = array();
			
		KalturaLog::info("Loading configuration file at: " . date('Y-m-d H:i'));
		
		$configItems = $this->createConfigItem($this->schedulerConfig->toArray());
		$taskConfigs = $this->schedulerConfig->getTaskConfigList();
		
		$this->logDir = $this->schedulerConfig->getLogDir();
		$this->statusInterval = $this->schedulerConfig->getStatusInterval();
		$this->schedulerStatusInterval = $this->schedulerConfig->getSchedulerStatusInterval();
		KDwhClient::setEnabled($this->schedulerConfig->getDwhEnabled());
		KDwhClient::setFileName($this->schedulerConfig->getDwhPath());
		
		$taskConfigsValidations = array();
		foreach($taskConfigs as $taskConfig)
		{
			/* @var $taskConfig KSchedularTaskConfig */
			
			if(is_null($taskConfig->type)) // is the scheduler itself
				continue;
				
			if(isset($taskConfigsValidations[$taskConfig->id]))
			{
				KalturaLog::err("Duplicated worker id [$taskConfig->id] in worker names [$taskConfig->name] and [" . $taskConfigsValidations[$taskConfig->id] . "]");
				$this->keepRunning = false;
				return;
			}
			
			if(in_array($taskConfig->name, $taskConfigsValidations))
			{
				KalturaLog::err("Duplicated worker name [$taskConfig->name] in worker ids [$taskConfig->id] and [" . array_search($taskConfig->name, $taskConfigsValidations) . "]");
				$this->keepRunning = false;
				return;
			}
			
			$taskConfigsValidations[$taskConfig->id] = $taskConfig->name;
			$vars = get_object_vars($taskConfig);
			$subConfigItems = $this->createConfigItem($vars, $taskConfig->id, $taskConfig->name);
			$configItems = array_merge($configItems, $subConfigItems);
		}
		KalturaLog::info("sending configuration to the server");<commit_msg>qnd: send the workers config as Zend_Config array item<commit_after><?php
/**
 * Will schedual execution of external commands
 * Copied base functionality from KScheduler
 *
 * @package Scheduler
 */
class KGenericScheduler
{
	/**
	 * @var bool
	 */
	private $enableDebug = true;
	
	/**
	 * @var KSchedulerConfig
	 */
	private $schedulerConfig = null;
	
	/**
	 * @var bool
	 */
	private $keepRunning = true;
	
	private $logDir = "c:/web/kaltura/log";
	private $phpPath = null;
	private $statusInterval;
	private $schedulerStatusInterval;
	private $nextStatusTime = 0;
	private $nextSchedulerStatusTime = 0;
	
	/**
	 * Stores all groups of tasks, the index is the task type
	 * @var array is an array of array
	 */
	private $runningTasks = array();
	
	/**
	 * Stores the last execution time of each task type
	 * @var array
	 */
	private $lastRunTime = array();
	
	/**
	 * Stores the last executed index of each task type
	 * @var array
	 */
	private $nextRunIndex = array();
	
	/**
	 * Stores the size of the queue on the server of each task type
	 * @var array
	 */
	private $queueSizes = array();
	
	/**
	 * @param string $phpPath
	 * @param string $configFileName
	 */
	public function __construct($phpPath, $configFileName)
	{
		$this->phpPath = $phpPath;
		$this->loadConfig($configFileName);
	}
	
	public function __destruct()
	{
		$this->_cleanup();
	}
	
	private function initAllWorkers()
	{
		$taskConfigs = $this->schedulerConfig->getTaskConfigList();
		
		foreach($taskConfigs as $taskConfig)
		{
			if(!$taskConfig->type)
				continue;
			
			$tmpConfig = clone $taskConfig;
			$tmpConfig->setInitOnly(true);
			
			KalturaLog::info('Initilizing ' . $tmpConfig->name);
			$tasksetPath = $this->schedulerConfig->getTasksetPath();
			$proc = new KProcessWrapper(0, $this->logDir, $this->phpPath, $tasksetPath, $tmpConfig);
			sleep(1);
		}
	}
	
	private function loadConfig($configFileName = null)
	{
		$firstLoad = is_null($this->schedulerConfig);
		if($firstLoad)
		{
			$this->schedulerConfig = new KSchedulerConfig($configFileName);
			date_default_timezone_set($this->schedulerConfig->getTimezone());
			KalturaLog::info(file_get_contents('VERSION.txt'));
		}
		else
		{	
			if(!$this->schedulerConfig->reloadRequired())
				return;
				
			sleep(2); // make sure the file finsied to be written
			$this->schedulerConfig->load();
		}
		
		KScheduleHelperManager::clearFilters();
		$this->queueSizes = array();
			
		KalturaLog::info("Loading configuration file at: " . date('Y-m-d H:i'));
		
		$configItems = $this->createConfigItem($this->schedulerConfig->toArray());
		$taskConfigs = $this->schedulerConfig->getTaskConfigList();
		
		$this->logDir = $this->schedulerConfig->getLogDir();
		$this->statusInterval = $this->schedulerConfig->getStatusInterval();
		$this->schedulerStatusInterval = $this->schedulerConfig->getSchedulerStatusInterval();
		KDwhClient::setEnabled($this->schedulerConfig->getDwhEnabled());
		KDwhClient::setFileName($this->schedulerConfig->getDwhPath());
		
		$taskConfigsValidations = array();
		foreach($taskConfigs as $taskConfig)
		{
			/* @var $taskConfig KSchedularTaskConfig */
			
			if(is_null($taskConfig->type)) // is the scheduler itself
				continue;
				
			if(isset($taskConfigsValidations[$taskConfig->id]))
			{
				KalturaLog::err("Duplicated worker id [$taskConfig->id] in worker names [$taskConfig->name] and [" . $taskConfigsValidations[$taskConfig->id] . "]");
				$this->keepRunning = false;
				return;
			}
			
			if(in_array($taskConfig->name, $taskConfigsValidations))
			{
				KalturaLog::err("Duplicated worker name [$taskConfig->name] in worker ids [$taskConfig->id] and [" . array_search($taskConfig->name, $taskConfigsValidations) . "]");
				$this->keepRunning = false;
				return;
			}
			
			$taskConfigsValidations[$taskConfig->id] = $taskConfig->name;
			$subConfigItems = $this->createConfigItem($taskConfig->toArray(), $taskConfig->id, $taskConfig->name);
			$configItems = array_merge($configItems, $subConfigItems);
		}
		KalturaLog::info("sending configuration to the server");
{'fraction_non_alphanumeric': 0.14673577687276318, 'fraction_numerical': 0.001234110823151919, 'mean_word_length': 4.208226221079691, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 114, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5384913', 'n_tokens_mistral': 2916, 'n_tokens_neox': 2512, 'n_words': 531}