<commit_before><?php

/*
 * This file is part of the ONGR package.
 *
 * (c) NFQ Technologies UAB <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace ONGR\TranslationsBundle\Storage;

use ONGR\ElasticsearchBundle\DSL\Filter\TermsFilter;
use ONGR\ElasticsearchBundle\DSL\Query\MatchAllQuery;
use ONGR\ElasticsearchBundle\ORM\Manager;
use ONGR\ElasticsearchBundle\ORM\Repository;
use ONGR\TranslationsBundle\Document\Message;
use ONGR\TranslationsBundle\Document\Translation;

/**
 * Elasticsearch storage for translations.
 */
class ElasticsearchStorage implements StorageInterface
{
    /**
     * @var Repository Elasticsearch repository used for storing translations.
     */
    private $repository;

    /**
     * Injects elasticsearch repository for storage actions.
     *
     * @param Repository $repository Elasticsearch repository.
     */
    public function __construct(Repository $repository)
    {
        $this->repository = $repository;
    }

    /**
     * {@inheritdoc}
     */
    public function read($locales = [], $domains = [])
    {
        $search = $this
            ->getRepository()
            ->createSearch()
            ->setScroll('2m')
            ->addQuery(new MatchAllQuery());

        if (!empty($locales)) {
            $search->addFilter(new TermsFilter('locale', $locales));
        }

        if (!empty($domains)) {
            $search->addFilter(new TermsFilter('domain', $domains));
        }

        return $this->getRepository()->execute($search, Repository::RESULTS_OBJECT);
    }

    /**
     * {@inheritdoc}
     */
    public function write(array $translations)
    {
        foreach ($translations as $path => $domains) {
            foreach ($domains as $domain => $transMeta) {
                foreach ($transMeta['translations'] as $key => $keyTrans) {
                    $document = $this->getRepository()->createDocument();
                    $document->setDomain($domain);
                    $document->setKey($key);
                    $document->setPath($path);
                    $document->setFormat($transMeta['format']);

                    foreach ($keyTrans as $locale => $trans) {
                        $message = new Message();
                        $message->setLocale($locale);
                        $message->setMessage($trans);
                        $document->addMessage($message);
                    }
                    $this->getRepository()->getManager()->persist($document);
                }
            }
        }

        $this->getRepository()->getManager()->commit();
    }

    /**
     * Returns repository for translations.
     *
     * @return Repository
     */
    private function getRepository()
    {
        return $this->repository;
    }
}<commit_msg>storage can now also parse real documents<commit_after><?php

/*
 * This file is part of the ONGR package.
 *
 * (c) NFQ Technologies UAB <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace ONGR\TranslationsBundle\Storage;

use ONGR\ElasticsearchBundle\DSL\Filter\TermsFilter;
use ONGR\ElasticsearchBundle\DSL\Query\MatchAllQuery;
use ONGR\ElasticsearchBundle\ORM\Manager;
use ONGR\ElasticsearchBundle\ORM\Repository;
use ONGR\TranslationsBundle\Document\Message;
use ONGR\TranslationsBundle\Document\Translation;

/**
 * Elasticsearch storage for translations.
 */
class ElasticsearchStorage implements StorageInterface
{
    /**
     * @var Repository Elasticsearch repository used for storing translations.
     */
    private $repository;

    /**
     * Injects elasticsearch repository for storage actions.
     *
     * @param Repository $repository Elasticsearch repository.
     */
    public function __construct(Repository $repository)
    {
        $this->repository = $repository;
    }

    /**
     * {@inheritdoc}
     */
    public function read($locales = [], $domains = [])
    {
        $search = $this
            ->getRepository()
            ->createSearch()
            ->setScroll('2m')
            ->addQuery(new MatchAllQuery());

        if (!empty($locales)) {
            $search->addFilter(new TermsFilter('locale', $locales));
        }

        if (!empty($domains)) {
            $search->addFilter(new TermsFilter('domain', $domains));
        }

        return $this->getRepository()->execute($search, Repository::RESULTS_OBJECT);
    }

    /**
     * {@inheritdoc}
     */
    public function write(array $translations)
    {
        if (!(reset($translations) instanceof Translation)) {
            $translations = $this->toDocumentArray($translations);
        }

        foreach ($translations as $translation) {
            $this->getRepository()->getManager()->persist($translation);
        }

        $this->getRepository()->getManager()->commit();
    }

    /**
     * Converts arrays to documents.
     *
     * @param array $translations
     *
     * @return array
     */
    private function toDocumentArray(array $translations)
    {
        $out = [];

        foreach ($translations as $path => $domains) {
            foreach ($domains as $domain => $transMeta) {
                foreach ($transMeta['translations'] as $key => $keyTrans) {
                    $document = $this->getRepository()->createDocument();
                    $document->setDomain($domain);
                    $document->setKey($key);
                    $document->setPath($path);
                    $document->setFormat($transMeta['format']);

                    foreach ($keyTrans as $locale => $trans) {
                        $message = new Message();
                        $message->setLocale($locale);
                        $message->setMessage($trans);
                        $document->addMessage($message);
                    }
                    $out[] = $document;
                }
            }
        }

        return $out;
    }

    /**
     * {@inheritdoc}
     */
    private function getRepository()
    {
        return $this->repository;
    }
}
{'fraction_non_alphanumeric': 0.12742561448900389, 'fraction_numerical': 0.0003234152652005175, 'mean_word_length': 2.0801792828685257, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2953414', 'n_tokens_mistral': 1616, 'n_tokens_neox': 1558, 'n_words': 343}