<commit_before><?php

declare(strict_types=1);

namespace Fazland\ElasticaBundle\Elastica;

use Elastica;
use Elastica\ResultSet\BuilderInterface;
use Fazland\ElasticaBundle\Configuration\TypeConfig;
use Fazland\ElasticaBundle\Elastica\ResultSet\Builder;
use Fazland\ElasticaBundle\Event\Events;
use Fazland\ElasticaBundle\Event\TypePopulateEvent;
use Fazland\ElasticaBundle\Index\MappingBuilder;
use Fazland\ElasticaBundle\Provider\NullProvider;
use Fazland\ElasticaBundle\Provider\ProviderInterface;
use Fazland\ElasticaBundle\Transformer\ElasticaToModelTransformerInterface;
use Fazland\ElasticaBundle\Transformer\ModelToElasticaTransformerInterface;
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
use Symfony\Component\OptionsResolver\OptionsResolver;

class Type extends Elastica\Type
{
    /**
     * @var TypeConfig
     */
    private $typeConfig;

    /**
     * @var EventDispatcherInterface
     */
    private $eventDispatcher;

    /**
     * @var ProviderInterface
     */
    private $provider;

    /**
     * @var ModelToElasticaTransformerInterface
     */
    private $modelTransformer;

    /**
     * @var ElasticaToModelTransformerInterface
     */
    private $elasticaTransformer;

    /**
     * @var Builder
     */
    private $defaultBuilder;

    public function __construct(Elastica\Index $index, TypeConfig $typeConfig)
    {
        parent::__construct($index, $typeConfig->getName());

        $this->typeConfig = $typeConfig;
    }

    public function buildMapping(): Elastica\Type\Mapping
    {
        $builder = $this->getMappingBuilder();
        $mapping = new Elastica\Type\Mapping();

        foreach ($builder->buildTypeMapping($this->typeConfig) as $name => $field) {
            $mapping->setParam($name, $field);
        }

        return $mapping;
    }

    public function sendMapping()
    {
        $this->setMapping($this->buildMapping());
    }

    public function populate(array $options = [])
    {
        $resolver = new OptionsResolver();
        $resolver->setDefaults([
            'offset' => null,
            'size' => null,
            'sleep' => null,
            'batch_size' => 100,
            'ignore_errors' => false,
        ]);
        $resolver->setAllowedTypes('offset', ['null', 'int']);
        $resolver->setAllowedTypes('size', ['null', 'int']);
        $resolver->setAllowedTypes('batch_size', ['null', 'int']);
        $resolver->setAllowedTypes('sleep', ['null', 'int']);

        $options = $resolver->resolve($options);
        $provider = $this->getProvider();

        $this->eventDispatcher->dispatch(Events::PRE_TYPE_POPULATE, new TypePopulateEvent($this));

        $i = 0;
        $objects = [];
        foreach ($provider->provide($options['offset'], $options['size']) as $object) {
            ++$i;
            $objects[] = $object;

            $this->eventDispatcher->dispatch(Events::TYPE_POPULATE, new TypePopulateEvent($this));

            if (count($objects) >= $options['batch_size']) {
                $this->persist(...$objects);
                $objects = [];

                $provider->clear();
            }

            if ($options['sleep']) {
                sleep($options['sleep']);
            }
        }

        if (count($objects) > 0) {
            $this->persist(...$objects);
            $this->provider->clear();
        }
<commit_msg>sleep every chunk, not every element<commit_after><?php

declare(strict_types=1);

namespace Fazland\ElasticaBundle\Elastica;

use Elastica;
use Elastica\ResultSet\BuilderInterface;
use Fazland\ElasticaBundle\Configuration\TypeConfig;
use Fazland\ElasticaBundle\Elastica\ResultSet\Builder;
use Fazland\ElasticaBundle\Event\Events;
use Fazland\ElasticaBundle\Event\TypePopulateEvent;
use Fazland\ElasticaBundle\Index\MappingBuilder;
use Fazland\ElasticaBundle\Provider\NullProvider;
use Fazland\ElasticaBundle\Provider\ProviderInterface;
use Fazland\ElasticaBundle\Transformer\ElasticaToModelTransformerInterface;
use Fazland\ElasticaBundle\Transformer\ModelToElasticaTransformerInterface;
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
use Symfony\Component\OptionsResolver\OptionsResolver;

class Type extends Elastica\Type
{
    /**
     * @var TypeConfig
     */
    private $typeConfig;

    /**
     * @var EventDispatcherInterface
     */
    private $eventDispatcher;

    /**
     * @var ProviderInterface
     */
    private $provider;

    /**
     * @var ModelToElasticaTransformerInterface
     */
    private $modelTransformer;

    /**
     * @var ElasticaToModelTransformerInterface
     */
    private $elasticaTransformer;

    /**
     * @var Builder
     */
    private $defaultBuilder;

    public function __construct(Elastica\Index $index, TypeConfig $typeConfig)
    {
        parent::__construct($index, $typeConfig->getName());

        $this->typeConfig = $typeConfig;
    }

    public function buildMapping(): Elastica\Type\Mapping
    {
        $builder = $this->getMappingBuilder();
        $mapping = new Elastica\Type\Mapping();

        foreach ($builder->buildTypeMapping($this->typeConfig) as $name => $field) {
            $mapping->setParam($name, $field);
        }

        return $mapping;
    }

    public function sendMapping()
    {
        $this->setMapping($this->buildMapping());
    }

    public function populate(array $options = [])
    {
        $resolver = new OptionsResolver();
        $resolver->setDefaults([
            'offset' => null,
            'size' => null,
            'sleep' => null,
            'batch_size' => 100,
            'ignore_errors' => false,
        ]);
        $resolver->setAllowedTypes('offset', ['null', 'int']);
        $resolver->setAllowedTypes('size', ['null', 'int']);
        $resolver->setAllowedTypes('batch_size', ['null', 'int']);
        $resolver->setAllowedTypes('sleep', ['null', 'int']);

        $options = $resolver->resolve($options);
        $provider = $this->getProvider();

        $this->eventDispatcher->dispatch(Events::PRE_TYPE_POPULATE, new TypePopulateEvent($this));

        $i = 0;
        $objects = [];
        foreach ($provider->provide($options['offset'], $options['size']) as $object) {
            ++$i;
            $objects[] = $object;

            $this->eventDispatcher->dispatch(Events::TYPE_POPULATE, new TypePopulateEvent($this));

            if (count($objects) >= $options['batch_size']) {
                $this->persist(...$objects);
                $objects = [];

                $provider->clear();

                if ($options['sleep']) {
                    sleep($options['sleep']);
                }
            }
        }

        if (count($objects) > 0) {
            $this->persist(...$objects);
            $this->provider->clear();
        }

{'fraction_non_alphanumeric': 0.14112962689868752, 'fraction_numerical': 0.0017696504940274296, 'mean_word_length': 2.677874186550976, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4401498', 'n_tokens_mistral': 1993, 'n_tokens_neox': 1918, 'n_words': 315}