<commit_before><?php
/**
 * @package framework.listener
 */
class f_listener_PublishListener
{

	
	/**
	 * @param f_persistentdocument_DocumentService $sender
	 * @param array $params
	 */
	public function onHourChange($sender, $params)
	{
		$date = $params['date'];
		if (Framework::isDebugEnabled())
		{
			Framework::debug(__METHOD__ . "($date)");
		}
		
		$start = date_Calendar::getInstance($date)->add(date_Calendar::HOUR, -1)->toString();
		$end = date_Calendar::getInstance($date)->add(date_Calendar::HOUR, 1)->toString();	
		$documentsArray = array_chunk($this->getDocumentIdsToProcess($start, $end), 500);
		foreach ($documentsArray as $chunk)
		{
			$processHandle = popen("php " . dirname(__FILE__) . DIRECTORY_SEPARATOR . "publishDocumentsBatch.php " . implode(" ", $chunk), "r");
			while ( ($string = fread($processHandle, 1000)) != false)
			{
				// do nothing
			}
			pclose($processHandle);
		}
	}
	
	private function getDocumentIdsToProcess($start, $end)
	{
		$toProcess = array();
		foreach (f_persistentdocument_PersistentDocumentModel::getDocumentModels() as $model)
		{
			if (strpos($model->getName(), 'modules_test/') === 0 || $model->publishOnDayChange() === false)
			{
				continue;
			}
			
			$pubproperty = $model->getProperty('publicationstatus');
			if (is_null($pubproperty))
			{
				return array();
			}
			
			$rc = RequestContext::getInstance();
			
			if ($model->isLocalized() && $pubproperty->isLocalized())
			{
				$langs = $rc->getSupportedLanguages();
			}
			else
			{
				$langs = array($rc->getDefaultLang());
			}
			
			foreach ($langs as $lang)
			{
				try
				{
					$rc->beginI18nWork($lang);
					$query = f_persistentdocument_PersistentProvider::getInstance()->createQuery($model->getName());
					$query->add(Restrictions::in('publicationstatus', array('ACTIVE', 'PUBLICATED')))->add(Restrictions::eq('model', $model->getName()))->add(Restrictions::orExp(Restrictions::between('startpublicationdate', $start, $end), Restrictions::between('endpublicationdate', $start, $end)))->setProjection(Projections::property('id', 'id'));
					$results = $query->find();
					foreach ($results as $resultArray)
					{
						$toProcess[] = $resultArray['id'] . '/' . $lang;
					}
					$rc->endI18nWork();
				}
				catch (Exception $e)
				{
					$rc->endI18nWork($e);
				}
			}
		}
		return $toProcess;
	}
}<commit_msg>[ENHANCE] publishlistener can now "fill time hole"<commit_after><?php
/**
 * @package framework.listener
 */
class f_listener_PublishListener
{

	
	/**
	 * @param f_persistentdocument_DocumentService $sender
	 * @param array $params
	 */
	public function onHourChange($sender, $params)
	{
		$date = $params['date'];
		if (Framework::isDebugEnabled())
		{
			Framework::debug(__METHOD__ . "($date,".var_export($params['previousRunTime'], true).")");
		}
		if (isset($params['previousRunTime']))
		{
			$start = date_Calendar::getInstanceFromTimestamp($params['previousRunTime'])->toString();
		}
		else
		{
			$start = date_Calendar::getInstance($date)->add(date_Calendar::HOUR, -1)->toString();
		}
		$end = date_Calendar::getInstance($date)->add(date_Calendar::HOUR, 1)->toString();	
		$documentsArray = array_chunk($this->getDocumentIdsToProcess($start, $end), 500);
		foreach ($documentsArray as $chunk)
		{
			$processHandle = popen("php " . dirname(__FILE__) . DIRECTORY_SEPARATOR . "publishDocumentsBatch.php " . implode(" ", $chunk), "r");
			while ( ($string = fread($processHandle, 1000)) != false)
			{
				// do nothing
			}
			pclose($processHandle);
		}
	}
	
	private function getDocumentIdsToProcess($start, $end)
	{
		$toProcess = array();
		foreach (f_persistentdocument_PersistentDocumentModel::getDocumentModels() as $model)
		{
			if (strpos($model->getName(), 'modules_test/') === 0 || $model->publishOnDayChange() === false)
			{
				continue;
			}
			
			$pubproperty = $model->getProperty('publicationstatus');
			if (is_null($pubproperty))
			{
				return array();
			}
			
			$rc = RequestContext::getInstance();
			
			if ($model->isLocalized() && $pubproperty->isLocalized())
			{
				$langs = $rc->getSupportedLanguages();
			}
			else
			{
				$langs = array($rc->getDefaultLang());
			}
			
			foreach ($langs as $lang)
			{
				try
				{
					$rc->beginI18nWork($lang);
					$query = f_persistentdocument_PersistentProvider::getInstance()->createQuery($model->getName());
					$query->add(Restrictions::in('publicationstatus', array('ACTIVE', 'PUBLICATED')))->add(Restrictions::eq('model', $model->getName()))->add(Restrictions::orExp(Restrictions::between('startpublicationdate', $start, $end), Restrictions::between('endpublicationdate', $start, $end)))->setProjection(Projections::property('id', 'id'));
					$results = $query->find();
					foreach ($results as $resultArray)
					{
						$toProcess[] = $resultArray['id'] . '/' . $lang;
					}
					$rc->endI18nWork();
				}
				catch (Exception $e)
				{
					$rc->endI18nWork($e);
				}
			}
		}
		return $toProcess;
	}
}
{'fraction_non_alphanumeric': 0.18452261306532663, 'fraction_numerical': 0.006432160804020101, 'mean_word_length': 4.205020920502092, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2117826', 'n_tokens_mistral': 1952, 'n_tokens_neox': 1604, 'n_words': 248}