<commit_before><?php
    /*********************************************************************************
     * Zurmo is a customer relationship management program developed by
     * Zurmo, Inc. Copyright (C) 2014 Zurmo Inc.
     *
     * Zurmo is free software; you can redistribute it and/or modify it under
     * the terms of the GNU Affero General Public License version 3 as published by the
     * Free Software Foundation with the addition of the following permission added
     * to Section 15 as permitted in Section 7(a): FOR ANY PART OF THE COVERED WORK
     * IN WHICH THE COPYRIGHT IS OWNED BY ZURMO, ZURMO DISCLAIMS THE WARRANTY
     * OF NON INFRINGEMENT OF THIRD PARTY RIGHTS.
     *
     * Zurmo is distributed in the hope that it will be useful, but WITHOUT
     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     * FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
     * details.
     *
     * You should have received a copy of the GNU Affero General Public License along with
     * this program; if not, see http://www.gnu.org/licenses or write to the Free
     * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
     * 02110-1301 USA.
     *
     * You can contact Zurmo, Inc. with a mailing address at 27 North Wacker Drive
     * Suite 370 Chicago, IL 60606. or at email address <EMAIL>.
     *
     * The interactive user interfaces in original and modified versions
     * of this program must display Appropriate Legal Notices, as required under
     * Section 5 of the GNU Affero General Public License version 3.
     *
     * In accordance with Section 7(b) of the GNU Affero General Public License version 3,
     * these Appropriate Legal Notices must retain the display of the Zurmo
     * logo and Zurmo copyright notice. If the display of the logo is not reasonably
     * feasible for technical reasons, the Appropriate Legal Notices must display the words
     * "Copyright Zurmo Inc. 2014. All rights reserved".
     ********************************************************************************/

    /**
     * For exports with many records we create jobs that will generate export file
     * in background, and send notification to user with export download link,
     * when export job is completed.
     */
    class ExportJob extends BaseJob
    {
        /**
         * Incremented as each model is processed. Utilized to determine the max models processing and when
         * that has been reached.
         * @var int
         */
        protected $totalModelsProcessed = 0;

        /**
         * @see BaseJob::$loadJobQueueOnCleanupAndFallback
         * @var bool
         */
        protected static $loadJobQueueOnCleanupAndFallback = true;

        /**
         * @returns Translated label that describes this job type.
         */
        public static function getDisplayName()
        {
           return Zurmo::t('ExportModule', 'Export Job');
        }

        /**
         * @return The type of the NotificationRules
         */
        public static function getType()
        {
            return 'Export';
        }

        /**
         * @return string
         */
        public static function getRecommendedRunFrequencyContent()
        {
            return Zurmo::t('ExportModule', 'Every 2 minutes.');
        }

        /**
         * @return int
         */
        public function getTotalModelsProcessed()
        {
            return $this->totalModelsProcessed;
        }

        /*
         * Run uncompleted export items and create export files
         */
        public function run()
        {
            $cancelledItems = ExportItem::getCancelledItems();
            foreach ($cancelledItems as $item)
            {
                $item->delete();
                $message = Zurmo::t('ExportModule', 'Deleted export item with ID: {id}', array('{id}' => $item->id));
                $this->getMessageLogger()->addInfoMessage($message);
            }
            $exportItems    = ExportItem::getUncompletedItems();
            $startTime      = Yii::app()->performance->startClock();
            Yii::app()->performance->startMemoryUsageMarker();
            if (count($exportItems) > 0)
            {
                foreach ($exportItems as $exportItem)
                {
                    $originalUser               = Yii::app()->user->userModel;
                    Yii::app()->user->userModel = $exportItem->owner;
                    $message = Zurmo::t('ExportModule', 'run: Beginning processing of export item with ID: {id} ', array('{id}' => $exportItem->id));
                    $this->getMessageLogger()->addInfoMessage($message);
                    try
                    {
                        //Update the job run status for the export item.
                        $exportItem->isJobRunning = true;
                        $exportItem->save();
                        $message = Zurmo::t('ExportModule', 'Update the job running status for export item with ID: {id} to true', array('{id}' => $exportItem->id));
                        $this->getMessageLogger()->addInfoMessage($message);
                        $this->processExportItem($exportItem);
                    }
                    catch (SecurityException $e)
                    {
                        $message = Zurmo::t('ExportModule', 'Export Item could not be processed due a SecurityException ' . $e->getMessage());
                        $this->getMessageLogger()->addInfoMessage($message);
                        $exportItem->isJobRunning = false;
                        $exportItem->save();
                        $this->processCompletedWithSecurityExceptionExportItem($exportItem);
                    }
                    $exportItem->isJobRunning = false;
                    $exportItem->save();
                    $message = Zurmo::t('ExportModule', 'Update the job running status for export item with ID: {id} to false', array('{id}' => $exportItem->id));
                    $this->getMessageLogger()->addInfoMessage($message);
                    if ($this->hasReachedMaximumProcessingCount())
                    {
                        $this->addMaximumProcessingCountMessageForAllExportItems();
                        break;
                    }
                    if (!Yii::app()->performance->isMemoryUsageSafe())
                    {
                        $this->addMaximumMemoryUsageReachedForAllExportItems();
                        break;
                    }
                }
                Yii::app()->user->userModel = $originalUser;
                if ($this->hasReachedMaximumProcessingCount() || !Yii::app()->performance->isMemoryUsageSafe())
                {
                    Yii::app()->jobQueue->add('Export', 5);
                }
            }
            $this->processEndMemoryUsageMessage((int)$startTime);
            return true;
        }

        /**
         * @param ExportItem $exportItem
         */
        protected function processExportItem(ExportItem $exportItem)
        {
            $dataProviderOrIdsToExport = unserialize($exportItem->serializedData);
            if (!is_array($dataProviderOrIdsToExport))
            {
                $exportItem->delete();
                return;
            }
            if ($dataProviderOrIdsToExport instanceOf RedBeanModelDataProvider)
            {
                $this->processRedBeanModelDataProviderExport($exportItem, $dataProviderOrIdsToExport);
            }
            elseif ($dataProviderOrIdsToExport instanceOf ReportDataProvider)
            {
                if ($dataProviderOrIdsToExport instanceOf MatrixReportDataProvider)
                {
                    $this->processMatrixReportDataProviderExport($exportItem, $dataProviderOrIdsToExport);
                }
                else
                {
                    $this->processReportDataProviderExport($exportItem, $dataProviderOrIdsToExport);
                }
            }
            else
            {
                $this->processIdsToExport($exportItem, $dataProviderOrIdsToExport);
            }
            unset($dataProviderOrIdsToExport);
        }

        /**
         * @param ExportItem $exportItem
         * @param RedBeanModelDataProvider $dataProvider
         */
        protected function processRedBeanModelDataProviderExport(ExportItem $exportItem, RedBeanModelDataProvider $dataProvider)
        {
            $headerData = array();
            $data       = array();
            $dataProvider->getPagination()->setPageSize($this->getAsynchronousPageSize());
            $offset     = (int)$exportItem->processOffset;
            $exportCompleted     = true;
            $startingMemoryUsage = memory_get_usage();
            while (true === $this->processExportPage($dataProvider, (int)$offset, $headerData, $data,
                                                    ($exportItem->exportFileModel->id < 0)))<commit_msg>Fixed broken ExportJob related tests<commit_after><?php
    /*********************************************************************************
     * Zurmo is a customer relationship management program developed by
     * Zurmo, Inc. Copyright (C) 2014 Zurmo Inc.
     *
     * Zurmo is free software; you can redistribute it and/or modify it under
     * the terms of the GNU Affero General Public License version 3 as published by the
     * Free Software Foundation with the addition of the following permission added
     * to Section 15 as permitted in Section 7(a): FOR ANY PART OF THE COVERED WORK
     * IN WHICH THE COPYRIGHT IS OWNED BY ZURMO, ZURMO DISCLAIMS THE WARRANTY
     * OF NON INFRINGEMENT OF THIRD PARTY RIGHTS.
     *
     * Zurmo is distributed in the hope that it will be useful, but WITHOUT
     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     * FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
     * details.
     *
     * You should have received a copy of the GNU Affero General Public License along with
     * this program; if not, see http://www.gnu.org/licenses or write to the Free
     * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
     * 02110-1301 USA.
     *
     * You can contact Zurmo, Inc. with a mailing address at 27 North Wacker Drive
     * Suite 370 Chicago, IL 60606. or at email address <EMAIL>.
     *
     * The interactive user interfaces in original and modified versions
     * of this program must display Appropriate Legal Notices, as required under
     * Section 5 of the GNU Affero General Public License version 3.
     *
     * In accordance with Section 7(b) of the GNU Affero General Public License version 3,
     * these Appropriate Legal Notices must retain the display of the Zurmo
     * logo and Zurmo copyright notice. If the display of the logo is not reasonably
     * feasible for technical reasons, the Appropriate Legal Notices must display the words
     * "Copyright Zurmo Inc. 2014. All rights reserved".
     ********************************************************************************/

    /**
     * For exports with many records we create jobs that will generate export file
     * in background, and send notification to user with export download link,
     * when export job is completed.
     */
    class ExportJob extends BaseJob
    {
        /**
         * Incremented as each model is processed. Utilized to determine the max models processing and when
         * that has been reached.
         * @var int
         */
        protected $totalModelsProcessed = 0;

        /**
         * @see BaseJob::$loadJobQueueOnCleanupAndFallback
         * @var bool
         */
        protected static $loadJobQueueOnCleanupAndFallback = true;

        /**
         * @returns Translated label that describes this job type.
         */
        public static function getDisplayName()
        {
           return Zurmo::t('ExportModule', 'Export Job');
        }

        /**
         * @return The type of the NotificationRules
         */
        public static function getType()
        {
            return 'Export';
        }

        /**
         * @return string
         */
        public static function getRecommendedRunFrequencyContent()
        {
            return Zurmo::t('ExportModule', 'Every 2 minutes.');
        }

        /**
         * @return int
         */
        public function getTotalModelsProcessed()
        {
            return $this->totalModelsProcessed;
        }

        /*
         * Run uncompleted export items and create export files
         */
        public function run()
        {
            $cancelledItems = ExportItem::getCancelledItems();
            foreach ($cancelledItems as $item)
            {
                $item->delete();
                $message = Zurmo::t('ExportModule', 'Deleted export item with ID: {id}', array('{id}' => $item->id));
                $this->getMessageLogger()->addInfoMessage($message);
            }
            $exportItems    = ExportItem::getUncompletedItems();
            $startTime      = Yii::app()->performance->startClock();
            Yii::app()->performance->startMemoryUsageMarker();
            if (count($exportItems) > 0)
            {
                foreach ($exportItems as $exportItem)
                {
                    $originalUser               = Yii::app()->user->userModel;
                    Yii::app()->user->userModel = $exportItem->owner;
                    $message = Zurmo::t('ExportModule', 'run: Beginning processing of export item with ID: {id} ', array('{id}' => $exportItem->id));
                    $this->getMessageLogger()->addInfoMessage($message);
                    try
                    {
                        //Update the job run status for the export item.
                        $exportItem->isJobRunning = true;
                        $exportItem->save();
                        $message = Zurmo::t('ExportModule', 'Update the job running status for export item with ID: {id} to true', array('{id}' => $exportItem->id));
                        $this->getMessageLogger()->addInfoMessage($message);
                        $this->processExportItem($exportItem);
                    }
                    catch (SecurityException $e)
                    {
                        $message = Zurmo::t('ExportModule', 'Export Item could not be processed due a SecurityException ' . $e->getMessage());
                        $this->getMessageLogger()->addInfoMessage($message);
                        $exportItem->isJobRunning = false;
                        $exportItem->save();
                        $this->processCompletedWithSecurityExceptionExportItem($exportItem);
                    }
                    $exportItem->isJobRunning = false;
                    $exportItem->save();
                    $message = Zurmo::t('ExportModule', 'Update the job running status for export item with ID: {id} to false', array('{id}' => $exportItem->id));
                    $this->getMessageLogger()->addInfoMessage($message);
                    if ($this->hasReachedMaximumProcessingCount())
                    {
                        $this->addMaximumProcessingCountMessageForAllExportItems();
                        break;
                    }
                    if (!Yii::app()->performance->isMemoryUsageSafe())
                    {
                        $this->addMaximumMemoryUsageReachedForAllExportItems();
                        break;
                    }
                }
                Yii::app()->user->userModel = $originalUser;
                if ($this->hasReachedMaximumProcessingCount() || !Yii::app()->performance->isMemoryUsageSafe())
                {
                    Yii::app()->jobQueue->add('Export', 5);
                }
            }
            $this->processEndMemoryUsageMessage((int)$startTime);
            return true;
        }

        /**
         * @param ExportItem $exportItem
         */
        protected function processExportItem(ExportItem $exportItem)
        {
            $dataProviderOrIdsToExport = unserialize($exportItem->serializedData);
            if ($dataProviderOrIdsToExport instanceOf RedBeanModelDataProvider)
            {
                $this->processRedBeanModelDataProviderExport($exportItem, $dataProviderOrIdsToExport);
            }
            elseif ($dataProviderOrIdsToExport instanceOf ReportDataProvider)
            {
                if ($dataProviderOrIdsToExport instanceOf MatrixReportDataProvider)
                {
                    $this->processMatrixReportDataProviderExport($exportItem, $dataProviderOrIdsToExport);
                }
                else
                {
                    $this->processReportDataProviderExport($exportItem, $dataProviderOrIdsToExport);
                }
            }
            else if (is_array($dataProviderOrIdsToExport))
            {
                $this->processIdsToExport($exportItem, $dataProviderOrIdsToExport);
            }
            unset($dataProviderOrIdsToExport);
        }

        /**
         * @param ExportItem $exportItem
         * @param RedBeanModelDataProvider $dataProvider
         */
        protected function processRedBeanModelDataProviderExport(ExportItem $exportItem, RedBeanModelDataProvider $dataProvider)
        {
            $headerData = array();
            $data       = array();
            $dataProvider->getPagination()->setPageSize($this->getAsynchronousPageSize());
            $offset     = (int)$exportItem->processOffset;
            $exportCompleted     = true;
            $startingMemoryUsage = memory_get_usage();
            while (true === $this->processExportPage($dataProvider, (int)$offset, $headerData, $data,
                                                    ($exportItem->exportFileModel->id < 0)))
{'fraction_non_alphanumeric': 0.10827067669172932, 'fraction_numerical': 0.004678362573099415, 'mean_word_length': 1.8438390877415267, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 134, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6073179', 'n_tokens_mistral': 4295, 'n_tokens_neox': 4035, 'n_words': 1213}