<commit_before><?php

/*
 * This file is part of the Access to Memory (AtoM) software.
 *
 * Access to Memory (AtoM) is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Access to Memory (AtoM) is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Access to Memory (AtoM).  If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * A base for a Gearman job in AtoM.
 *
 * @author     <NAME> <<EMAIL>>
 */
class arBaseJob extends Net_Gearman_Job_Common
{
    /*
     * Parallel execution and retry time:
     *
     * In instances where two or more workers are setup, multiple jobs could run in parallel.
     * We want to avoid that in jobs that make sensitive changes to the nested set, like the
     * arObjectMoveJob and the import jobs. The Gearman job server doesn't include a built in
     * system to postpone/schedule jobs so, if multiple jobs from the $avoidParallelExecutionJobs
     * variable bellow are executed at the same time, the late ones will wait, retrying after
     * the amount of seconds indicated in $waitForRetryTime, until the previous ones are finished
     * or the maximun amount of tries ($maxTries) is reached. Due to the limitations of the Gearman
     * job server, the waiting jobs will block the workers executing them until they are ended.
     */
    protected $avoidParallelExecutionJobs = ['arObjectMoveJob', 'arFileImportJob'];
    protected $waitForRetryTime = 10;
    protected $maxRetries = 10;
    protected $dispatcher;
    /*
     * Required parameters:
     *
     * Declares parameters that are mandatory for the jobs execution. They can be
     * extended on each job subclasse using the $extraRequiredParameters property.
     * If any of the required paramareters is missing the job will fail.
     */
    private $requiredParameters = ['id', 'name'];

    public function run($parameters)
    {
        $context = sfContext::getInstance();
        $this->i18n = $context->i18n;
        $this->user = $context->user;
        $this->dispatcher = $context->getEventDispatcher();
        sfConfig::add(QubitSetting::getSettingsArray());

        $this->checkRequiredParameters($parameters);

        // Instantiate QubitJob
        if (null === $this->job = QubitJob::getById($parameters['id'])) {
            throw new Net_Gearman_Job_Exception('Called a Gearman worker with an invalid QubitJob id.');
        }

        $this->logger = new arJobLogger($this->dispatcher, ['level' => sfLogger::INFO, 'job' => $this->job]);

        // Catch all possible exceptions in job execution and throw
        // Net_Gearman_Job_Exception to avoid breaking the worker
        try {
            $this->info($this->i18n->__('Job started.'));

            // If this is a sensitive job
            if (in_array(get_class($this), $this->avoidParallelExecutionJobs)) {
                // Wait until other sensitive jobs are finished by order
                $retries = 0;
                while (!$this->canBeFullyExecuted()) {
                    // Fail the job if we have reached the max. amount of retries
                    if ($retries++ == $this->maxRetries) {
                        $this->error($this->i18n->__('Maximum retries reached (%1). Please, try to launch the job again when other sensitive jobs are finished or contact an administrator', ['%1' => $this->maxRetries]));

                        return false;
                    }

                    // Log retry info
                    $this->info($this->i18n->__('Another sensitive job is being executed, will retry in %1 seconds', ['%1' => $this->waitForRetryTime]));

                    sleep($this->waitForRetryTime);
                }
            }

            Qubit::clearClassCaches();

            // Attempt signIn based on job's user. Before calling signIn(), $this->
            // user->isAuthenticated() will always evaluate to false - user object is
            // assigned in signIn().
            $this->signIn();

            // Run un-authenticated job cleanup if this is an unauthenticated job.
            if (!$this->user->isAuthenticated()) {
                $this->deleteOldUnauthenticatedJobs();
            }

            $this->runJob($parameters);

            QubitSearch::getInstance()->flushBatch();

            $this->signOut();

            $this->info($this->i18n->__('Job finished.'));
        } catch (Exception $e) {
            // TODO: Create undoJob() functions in subclasses for cleanups

            // Mark QubitJob as failed
            $this->error('Exception: '.$e->getMessage());
        }
    }

    /**
     * Check if all required parameters are present in $parameters,
     * if one is missing throw an exception.
     *
     * @param $parameters the parameters passed to this job
     */
    protected function checkRequiredParameters($parameters)
    {
        if (isset($this->extraRequiredParameters)) {
            $this->requiredParameters = array_merge($this->requiredParameters, $this->extraRequiredParameters);
        }

        foreach ($this->requiredParameters as $paramName) {
            if (!isset($parameters[$paramName])) {
                throw new Net_Gearman_Job_Exception("Required parameter not found for job: {$paramName}");
            }
        }
    }

    /**
     * A wrapper to log error messages and set the QubitJob status to error.
     * This will also attach the error message as a note in the QubitJob.
     *
     * @param string $message the error message
     */
    protected function error($message)
    {<commit_msg>Improve job worker debug output, refs #13504<commit_after><?php

/*
 * This file is part of the Access to Memory (AtoM) software.
 *
 * Access to Memory (AtoM) is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Access to Memory (AtoM) is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Access to Memory (AtoM).  If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * A base for a Gearman job in AtoM.
 *
 * @author     <NAME> <<EMAIL>>
 */
class arBaseJob extends Net_Gearman_Job_Common
{
    /*
     * Parallel execution and retry time:
     *
     * In instances where two or more workers are setup, multiple jobs could run in parallel.
     * We want to avoid that in jobs that make sensitive changes to the nested set, like the
     * arObjectMoveJob and the import jobs. The Gearman job server doesn't include a built in
     * system to postpone/schedule jobs so, if multiple jobs from the $avoidParallelExecutionJobs
     * variable bellow are executed at the same time, the late ones will wait, retrying after
     * the amount of seconds indicated in $waitForRetryTime, until the previous ones are finished
     * or the maximun amount of tries ($maxTries) is reached. Due to the limitations of the Gearman
     * job server, the waiting jobs will block the workers executing them until they are ended.
     */
    protected $avoidParallelExecutionJobs = ['arObjectMoveJob', 'arFileImportJob'];
    protected $waitForRetryTime = 10;
    protected $maxRetries = 10;
    protected $dispatcher;
    /*
     * Required parameters:
     *
     * Declares parameters that are mandatory for the jobs execution. They can be
     * extended on each job subclasse using the $extraRequiredParameters property.
     * If any of the required paramareters is missing the job will fail.
     */
    private $requiredParameters = ['id', 'name'];

    public function run($parameters)
    {
        $context = sfContext::getInstance();
        $this->i18n = $context->i18n;
        $this->user = $context->user;
        $this->dispatcher = $context->getEventDispatcher();
        sfConfig::add(QubitSetting::getSettingsArray());

        $this->checkRequiredParameters($parameters);

        // Instantiate QubitJob
        if (null === $this->job = QubitJob::getById($parameters['id'])) {
            throw new Net_Gearman_Job_Exception('Called a Gearman worker with an invalid QubitJob id.');
        }

        $this->logger = new arJobLogger($this->dispatcher, ['level' => sfLogger::INFO, 'job' => $this->job]);

        // Catch all possible exceptions in job execution and throw
        // Net_Gearman_Job_Exception to avoid breaking the worker
        try {
            $this->info($this->i18n->__('Job started.'));

            // If this is a sensitive job
            if (in_array(get_class($this), $this->avoidParallelExecutionJobs)) {
                // Wait until other sensitive jobs are finished by order
                $retries = 0;
                while (!$this->canBeFullyExecuted()) {
                    // Fail the job if we have reached the max. amount of retries
                    if ($retries++ == $this->maxRetries) {
                        $this->error($this->i18n->__('Maximum retries reached (%1). Please, try to launch the job again when other sensitive jobs are finished or contact an administrator', ['%1' => $this->maxRetries]));

                        return false;
                    }

                    // Log retry info
                    $this->info($this->i18n->__('Another sensitive job is being executed, will retry in %1 seconds', ['%1' => $this->waitForRetryTime]));

                    sleep($this->waitForRetryTime);
                }
            }

            Qubit::clearClassCaches();

            // Attempt signIn based on job's user. Before calling signIn(), $this->
            // user->isAuthenticated() will always evaluate to false - user object is
            // assigned in signIn().
            $this->signIn();

            // Run un-authenticated job cleanup if this is an unauthenticated job.
            if (!$this->user->isAuthenticated()) {
                $this->deleteOldUnauthenticatedJobs();
            }

            $this->runJob($parameters);

            QubitSearch::getInstance()->flushBatch();

            $this->signOut();

            $this->info($this->i18n->__('Job finished.'));
        } catch (Throwable $e) {
            // TODO: Create undoJob() functions in subclasses for cleanups

            // Mark QubitJob as failed
            $this->error('Exception: '.$e->getMessage());
            $this->error('File: '.$e->getFile());
            $this->error('Line: '.$e->getLine());
        }
    }

    /**
     * Check if all required parameters are present in $parameters,
     * if one is missing throw an exception.
     *
     * @param $parameters the parameters passed to this job
     */
    protected function checkRequiredParameters($parameters)
    {
        if (isset($this->extraRequiredParameters)) {
            $this->requiredParameters = array_merge($this->requiredParameters, $this->extraRequiredParameters);
        }

        foreach ($this->requiredParameters as $paramName) {
            if (!isset($parameters[$paramName])) {
                throw new Net_Gearman_Job_Exception("Required parameter not found for job: {$paramName}");
            }
        }
    }

    /**
     * A wrapper to log error messages and set the QubitJob status to error.
     * This will also attach the error message as a note in the QubitJob.
     *
     * @param string $message the error message
     */
    protected function error($message)
    {
{'fraction_non_alphanumeric': 0.1016640450368408, 'fraction_numerical': 0.0040566272042387615, 'mean_word_length': 2.4114656876588536, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2615904', 'n_tokens_mistral': 3185, 'n_tokens_neox': 3070, 'n_words': 1170}