<commit_before><?php

/*
 * This file is part of the ONGR package.
 *
 * (c) NFQ Technologies UAB <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace ONGR\ElasticsearchBundle\Service;

use ONGR\ElasticsearchBundle\Service\Json\JsonReader;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Output\OutputInterface;

/**
 * ImportService class.
 */
class ImportService
{
    /**
     * Imports Elasticsearch index data.
     *
     * @param Manager         $manager
     * @param string          $filename
     * @param bool            $raw
     * @param OutputInterface $output
     * @param int             $bulkSize
     *
     * @throws \Exception
     */
    public function importIndex($manager, $filename, $raw, OutputInterface $output, $bulkSize = 1000)
    {
        if (!$raw) {
            throw new \Exception('Currently only raw import is supported. Please set --raw flag to use it.');
        }

        $this->executeRawImport($manager, $this->getFilePath($filename), $output, $bulkSize);
    }

    /**
     * Executes a raw import.
     *
     * @param Manager         $manager
     * @param string          $filename
     * @param OutputInterface $output
     * @param int             $bulkSize
     */
    protected function executeRawImport(Manager $manager, $filename, OutputInterface $output, $bulkSize)
    {
        $reader = $this->getReader($manager, $filename, false);

        $progress = new ProgressBar($output, $reader->count());
        $progress->setRedrawFrequency(100);
        $progress->start();

        foreach ($reader as $key => $document) {
            $data = $document['_source'];
            $data['_id'] = $document['_id'];

            $manager->bulk('index', $document['_type'], $data);

            if (($key + 1) % $bulkSize == 0) {
                $manager->commit();
            }

            $progress->advance();
        }

        if (($key + 1) % $bulkSize != 0) {
            $manager->commit();
        }

        $progress->finish();
        $output->writeln('');
    }<commit_msg>commit now has a protection if bulk is empty<commit_after><?php

/*
 * This file is part of the ONGR package.
 *
 * (c) NFQ Technologies UAB <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace ONGR\ElasticsearchBundle\Service;

use ONGR\ElasticsearchBundle\Service\Json\JsonReader;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Output\OutputInterface;

/**
 * ImportService class.
 */
class ImportService
{
    /**
     * Imports Elasticsearch index data.
     *
     * @param Manager         $manager
     * @param string          $filename
     * @param bool            $raw
     * @param OutputInterface $output
     * @param int             $bulkSize
     *
     * @throws \Exception
     */
    public function importIndex($manager, $filename, $raw, OutputInterface $output, $bulkSize = 1000)
    {
        if (!$raw) {
            throw new \Exception('Currently only raw import is supported. Please set --raw flag to use it.');
        }

        $this->executeRawImport($manager, $this->getFilePath($filename), $output, $bulkSize);
    }

    /**
     * Executes a raw import.
     *
     * @param Manager         $manager
     * @param string          $filename
     * @param OutputInterface $output
     * @param int             $bulkSize
     */
    protected function executeRawImport(Manager $manager, $filename, OutputInterface $output, $bulkSize)
    {
        $reader = $this->getReader($manager, $filename, false);

        $progress = new ProgressBar($output, $reader->count());
        $progress->setRedrawFrequency(100);
        $progress->start();

        foreach ($reader as $key => $document) {
            $data = $document['_source'];
            $data['_id'] = $document['_id'];

            $manager->bulk('index', $document['_type'], $data);

            if (($key + 1) % $bulkSize == 0) {
                $manager->commit();
            }

            $progress->advance();
        }

        $manager->commit();

        $progress->finish();
        $output->writeln('');
    }
{'fraction_non_alphanumeric': 0.1329935125115848, 'fraction_numerical': 0.004633920296570899, 'mean_word_length': 2.224047796863331, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1428344', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1184, 'n_words': 317}