<commit_before><?php

namespace Concrete\Job;

use Concrete\Core\Database\Connection\Connection;
use Concrete\Core\Entity\Express\Entity;
use Concrete\Core\Entity\Express\Entry;
use Concrete\Core\Entity\Site\Site;
use Concrete\Core\Express\ObjectManager;
use Concrete\Core\Express\Search\Index\EntityIndex;
use Concrete\Core\File\File;
use Concrete\Core\Job\JobQueue;
use Concrete\Core\Job\JobQueueMessage;
use Concrete\Core\Job\QueueableJob;
use Concrete\Core\Page\Page;
use Concrete\Core\Search\Index\IndexManagerInterface;
use Concrete\Core\Search\Index\IndexObjectProvider;
use Concrete\Core\Support\Facade\Facade;
use Concrete\Core\User\User;
use Punic\Misc as PunicMisc;

class IndexSearchAll extends QueueableJob
{
    // A flag for clearing the index
    const CLEAR = '-1';
    const CLEAR_EXPRESS_ENTITY = '-2';

    public $jQueueBatchSize = 50;
    public $jNotUninstallable = 1;
    public $jSupportsQueue = true;

    /** @var array The result from the last queue item */
    protected $result;

    protected $clearTable = true;

    /*
     * @var \Concrete\Core\Search\Index\IndexManagerInterface
     */
    protected $indexManager;

    /**
     * @var \Concrete\Core\Database\Connection\Connection
     */
    protected $connection;

    /**
     * @var ObjectManager
     */
    protected $objectManager;

    /**
     * @var IndexObjectProvider
     */
    protected $dataProvider;

    public function getJobName()
    {
        return t('Index Search Engine - All');
    }

    public function getJobDescription()
    {
        return t('Empties the page search index and reindexes all pages.');
    }

    public function __construct(
        IndexManagerInterface $indexManager,
        Connection $connection,
        ObjectManager $objectManager,
        IndexObjectProvider $dataProvider)
    {
        $this->indexManager = $indexManager;
        $this->connection = $connection;
        $this->objectManager = $objectManager;
        $this->dataProvider = $dataProvider;
    }

    public function start(JobQueue $queue)
    {
        if ($this->clearTable) {
            // Send a "clear" queue item to clear out the index
            $queue->send(self::CLEAR);
        }

        // Queue everything
        foreach ($this->queueMessages() as $message) {
            $queue->send($message);
        }
    }

    /**
     * Messages to add to the queue.
     *
     * @return \Iterator
     */
    protected function queueMessages()
    {
        $pages = $users = $files = $sites = $objects = $entries = 0;

        foreach ($this->dataProvider->fetchExpressObjects() as $id) {
            yield self::CLEAR_EXPRESS_ENTITY . $id;<commit_msg>IndexSearchAll Job: wrap job queueing in transactions to improve performance and solve 500 Error for large sites<commit_after><?php

namespace Concrete\Job;

use Concrete\Core\Database\Connection\Connection;
use Concrete\Core\Entity\Express\Entity;
use Concrete\Core\Entity\Express\Entry;
use Concrete\Core\Entity\Site\Site;
use Concrete\Core\Express\ObjectManager;
use Concrete\Core\Express\Search\Index\EntityIndex;
use Concrete\Core\File\File;
use Concrete\Core\Job\JobQueue;
use Concrete\Core\Job\JobQueueMessage;
use Concrete\Core\Job\QueueableJob;
use Concrete\Core\Page\Page;
use Concrete\Core\Search\Index\IndexManagerInterface;
use Concrete\Core\Search\Index\IndexObjectProvider;
use Concrete\Core\Support\Facade\Facade;
use Concrete\Core\User\User;
use Punic\Misc as PunicMisc;

class IndexSearchAll extends QueueableJob
{
    // A flag for clearing the index
    const CLEAR = '-1';
    const CLEAR_EXPRESS_ENTITY = '-2';

    public $jQueueBatchSize = 50;
    public $jNotUninstallable = 1;
    public $jSupportsQueue = true;

    /** @var array The result from the last queue item */
    protected $result;

    protected $clearTable = true;

    /*
     * @var \Concrete\Core\Search\Index\IndexManagerInterface
     */
    protected $indexManager;

    /**
     * @var \Concrete\Core\Database\Connection\Connection
     */
    protected $connection;

    /**
     * @var ObjectManager
     */
    protected $objectManager;

    /**
     * @var IndexObjectProvider
     */
    protected $dataProvider;

    public function getJobName()
    {
        return t('Index Search Engine - All');
    }

    public function getJobDescription()
    {
        return t('Empties the page search index and reindexes all pages.');
    }

    public function __construct(
        IndexManagerInterface $indexManager,
        Connection $connection,
        ObjectManager $objectManager,
        IndexObjectProvider $dataProvider)
    {
        $this->indexManager = $indexManager;
        $this->connection = $connection;
        $this->objectManager = $objectManager;
        $this->dataProvider = $dataProvider;
    }

    public function start(JobQueue $queue)
    {
        if ($this->clearTable) {
            // Send a "clear" queue item to clear out the index
            $queue->send(self::CLEAR);
        }

        try {
            $i = 0;
            $transactionSize = 5000;
            foreach($this->queueMessages() as $i => $message) {
                if ($i % $transactionSize === 0)  {
                    $this->connection->beginTransaction();
                }

                $queue->send($message);

                if (($i+1) % $transactionSize === 0)  {
                    $this->connection->commit();
                }
            }

            if (($i+1) % $transactionSize !== 0)  {
                $this->connection->commit();
            }
        } catch (\Exception $e) {
            $this->connection->rollback();
            throw $e;
        }
    }

    /**
     * Messages to add to the queue.
     *
     * @return \Iterator
     */
    protected function queueMessages()
    {
        $pages = $users = $files = $sites = $objects = $entries = 0;

        foreach ($this->dataProvider->fetchExpressObjects() as $id) {
            yield self::CLEAR_EXPRESS_ENTITY . $id;
{'fraction_non_alphanumeric': 0.11367435640254096, 'fraction_numerical': 0.004179204279505183, 'mean_word_length': 2.4345579793340986, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2117445', 'n_tokens_mistral': 1753, 'n_tokens_neox': 1691, 'n_words': 401}