<commit_before><?php
/**
 * Going over all the entry vendor task items with:
 * service type = HUMAN
 * status pending OR processing
 * if the due date passed for more than overdueTimePercentage
 * update status to Error with description: Job Due Date was exceeded in $overdueTimePercentage %
 *
 * @package Scheduler
 * @subpackage ReachJobCleaner
 */
require_once('/opt/kaltura/app//alpha/lib/interfaces/IBaseObject.php');
require_once('/opt/kaltura/app/alpha/lib/interfaces/IIndexable.php');
require_once('/opt/kaltura/app/alpha/lib/interfaces/IRelatedObject.php');

class KAsyncReachJobCleaner extends KPeriodicWorker
{
	const JOB_DUE_DATE_EXCEEDED_MESSAGE = 'Job Due Date was exceeded in ';
	const MAX_PAGE_SIZE = 500;

	/*
	* @var KalturaReachClientPlugin
	*/
	private $reachClientPlugin = null;

	private $overdueTimePercentage = 100;

	/**
	 * @param KSchedularTaskConfig $taskConfig
	 */
	public function __construct($taskConfig = null)
	{
		parent::__construct($taskConfig);
		$this->reachClientPlugin = KalturaReachClientPlugin::get(self::$kClient);
		$overdueTimePercentage = $this->getAdditionalParams('overdueTimePercentage');
		$this->overdueTimePercentage = $overdueTimePercentage ? $overdueTimePercentage : 100;
	}

	/* (non-PHPdoc)
	 * @see KBatchBase::getType()
	 */
	public static function getType()
	{
		return KalturaBatchJobType::REACH_JOB_CLEANER;
	}

	/* (non-PHPdoc)
	 * @see KBatchBase::run()
	*/
	public function run($jobs = null)
	{
		KalturaLog::debug('Getting all the entry vendor task items');
		$filter = new KalturaEntryVendorTaskFilter();
		$filter->statusIn = KalturaEntryVendorTaskStatus::PENDING . ',' . KalturaEntryVendorTaskStatus::PROCESSING;
		$filter->expectedFinishTimeLessThanOrEqual = time();
		$filter->updatedAtGreaterThanOrEqual = time() -  (VendorServiceTurnAroundTime::TEN_DAYS + VendorServiceTurnAroundTime::FOUR_DAYS);
		$filter->orderBy = KalturaEntryVendorTaskOrderBy::UPDATED_AT_ASC;

		$pager = new KalturaFilterPager();
		$pager->pageSize = self::MAX_PAGE_SIZE;
		$pager->pageIndex = 1;

		$lastUpdatedAt = 0;
		$totalCount = 0;
		do {
			if ($lastUpdatedAt)
			{
				$filter->updatedAtGreaterThanOrEqual = $lastUpdatedAt;
			}
			try
			{
				$entryVendorTaskList = $this->reachClientPlugin->entryVendorTask->listAction($filter, $pager);
			}
			catch (Exception $e)
			{
				KalturaLog::debug('Could not list Entry Vendor Tasks ' . $e->getMessage());
				return;
			}

			$this->updateStuckJobs($entryVendorTaskList);
			$tasksCount = count($entryVendorTaskList->objects);
			$totalCount += $tasksCount;
			KalturaLog::debug("Handled More tasks - $tasksCount totalCount - " . $totalCount);
			if ($tasksCount)
			{
				$lastObject = end($entryVendorTaskList->objects);
				$lastUpdatedAt = $lastObject->updatedAt + 1;
			}

			unset($entryVendorTaskList);
			if (function_exists('gc_collect_cycles')) // php 5.3 and above
			{
				gc_collect_cycles();
			}
		} while ($tasksCount > 0);

		KalturaLog::debug('Done');
	}

	protected function updateStuckJobs($entryVendorTaskList)
	{
		foreach ($entryVendorTaskList->objects as $entryVendorTask)<commit_msg>REACH2-911:changing the loop to run on pages<commit_after><?php
/**
 * Going over all the entry vendor task items with:
 * service type = HUMAN
 * status pending OR processing
 * if the due date passed for more than overdueTimePercentage
 * update status to Error with description: Job Due Date was exceeded in $overdueTimePercentage %
 *
 * @package Scheduler
 * @subpackage ReachJobCleaner
 */
require_once('/opt/kaltura/app//alpha/lib/interfaces/IBaseObject.php');
require_once('/opt/kaltura/app/alpha/lib/interfaces/IIndexable.php');
require_once('/opt/kaltura/app/alpha/lib/interfaces/IRelatedObject.php');

class KAsyncReachJobCleaner extends KPeriodicWorker
{
	const JOB_DUE_DATE_EXCEEDED_MESSAGE = 'Job Due Date was exceeded in ';
	const MAX_PAGE_SIZE = 500;
	const MAX_MATCHES = 10000;

	/*
	* @var KalturaReachClientPlugin
	*/
	private $reachClientPlugin = null;

	private $overdueTimePercentage = 100;

	/**
	 * @param KSchedularTaskConfig $taskConfig
	 */
	public function __construct($taskConfig = null)
	{
		parent::__construct($taskConfig);
		$this->reachClientPlugin = KalturaReachClientPlugin::get(self::$kClient);
		$overdueTimePercentage = $this->getAdditionalParams('overdueTimePercentage');
		$this->overdueTimePercentage = $overdueTimePercentage ? $overdueTimePercentage : 100;
	}

	/* (non-PHPdoc)
	 * @see KBatchBase::getType()
	 */
	public static function getType()
	{
		return KalturaBatchJobType::REACH_JOB_CLEANER;
	}

	/* (non-PHPdoc)
	 * @see KBatchBase::run()
	*/
	public function run($jobs = null)
	{
		KalturaLog::debug('Getting all the entry vendor task items');
		$filter = new KalturaEntryVendorTaskFilter();
		$filter->statusIn = KalturaEntryVendorTaskStatus::PENDING . ',' . KalturaEntryVendorTaskStatus::PROCESSING;
		$filter->expectedFinishTimeLessThanOrEqual = time();
		$filter->updatedAtGreaterThanOrEqual = time() -  (VendorServiceTurnAroundTime::TEN_DAYS + VendorServiceTurnAroundTime::FOUR_DAYS);
		$filter->orderBy = KalturaEntryVendorTaskOrderBy::UPDATED_AT_ASC;

		$pager = new KalturaFilterPager();
		$pager->pageSize = self::MAX_PAGE_SIZE;
		$pager->pageIndex = 1;

		try
		{
			$entryVendorTaskList = $this->reachClientPlugin->entryVendorTask->listAction($filter, $pager);
		}
		catch (Exception $e)
		{
			KalturaLog::debug('Could not list Entry Vendor Tasks ' . $e->getMessage());
			return;
		}

		$totalCount = min($entryVendorTaskList->totalCount, self::MAX_MATCHES - 1);
		while ($totalCount > 0 && $pager->pageIndex <= 20)
		{
			$this->updateStuckJobs($entryVendorTaskList);

			$pager->pageIndex++;
			$totalCount = $totalCount - $pager->pageSize;
			$pager->pageSize = min(self::MAX_PAGE_SIZE, $totalCount);
			if ($pager->pageSize > 0)
			{
				try
				{
					$entryVendorTaskList = $this->reachClientPlugin->entryVendorTask->listAction($filter, $pager);
				}
				catch (Exception $e)
				{
					KalturaLog::debug('Could not list Entry Vendor Tasks ' . $e->getMessage());
					return;
				}
			}
		}

		KalturaLog::debug('Done');
	}

	protected function updateStuckJobs($entryVendorTaskList)
	{
		foreach ($entryVendorTaskList->objects as $entryVendorTask)
{'fraction_non_alphanumeric': 0.1278472890599936, 'fraction_numerical': 0.006416426050689766, 'mean_word_length': 4.938095238095238, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2399541', 'n_tokens_mistral': 2318, 'n_tokens_neox': 2076, 'n_words': 385}