<commit_before><?php

namespace Concrete\Core\Express\Export\EntryList;

use Concrete\Core\Attribute\MulticolumnTextExportableAttributeInterface;
use Concrete\Core\Entity\Attribute\Key\ExpressKey;
use Concrete\Core\Entity\Express\Entity;
use Concrete\Core\Entity\Express\Entry;
use Concrete\Core\Entity\Site\Site;
use Concrete\Core\Express\EntryList;
use Concrete\Core\Localization\Service\Date;
use Concrete\Core\Site\Service;
use League\Csv\Writer;

/**
 * A Writer class for Express Entry objects
 */
class CsvWriter
{

    /** @var Writer The writer we use to output */
    protected $writer;

    /**
     * @var Date
     */
    protected $dateFormatter;

    /**
     * @var string
     */
    private $datetime_format;

    /**
     * @var Service|null
     */
    private $siteService;

    public function __construct(Writer $writer, Date $dateFormatter, $datetime_format = DATE_ATOM)
    {
        $this->writer = $writer;
        $this->dateFormatter = $dateFormatter;
        $this->datetime_format = $datetime_format;
    }

    public function insertHeaders(Entity $entity)
    {
        $this->writer->insertOne(iterator_to_array($this->getHeaders($entity)));
    }

    /**
     * Insert all data from the passed EntryList
     * @param \Concrete\Core\Express\EntryList $list
     */
    public function insertEntryList(EntryList $list)
    {
        $list = clone $list;
        $this->writer->insertAll($this->projectList($list));
    }

    /**
     * A generator that takes an EntryList and converts it to CSV rows
     * @param \Concrete\Core\Express\EntryList $list
     * @return \Generator
     */
    private function projectList(EntryList $list)
    {
        $headers = array_keys(iterator_to_array($this->getHeaders($list->getEntity())));
        $statement = $list->deliverQueryObject()->execute();

        foreach ($statement as $result) {
            if ($entry = $list->getResult($result)) {
                yield $this->orderedEntry(iterator_to_array($this->projectEntry($entry)), $headers);
            }
        }
    }

    /**
     * Return an entry in proper order
     * @param array $entry
     * @param array $headerKeys
     *
     * @return array
     */
    private function orderedEntry(array $entry, array $headerKeys)
    {
        $result = [];

        foreach ($headerKeys as $key) {
            $result[$key] = $entry[$key];
        }

        return $result;
    }

    /**
     * Turn an Entry into an array
     * @param \Concrete\Core\Entity\Express\Entry $entry
     * @return array
     */
    private function projectEntry(Entry $entry)
    {
        $date = $entry->getDateCreated();<commit_msg>Making express csv export not error out with OOM<commit_after><?php

namespace Concrete\Core\Express\Export\EntryList;

use Concrete\Core\Attribute\MulticolumnTextExportableAttributeInterface;
use Concrete\Core\Entity\Attribute\Key\ExpressKey;
use Concrete\Core\Entity\Express\Entity;
use Concrete\Core\Entity\Express\Entry;
use Concrete\Core\Entity\Site\Site;
use Concrete\Core\Express\EntryList;
use Concrete\Core\Localization\Service\Date;
use Concrete\Core\Site\Service;
use Doctrine\ORM\EntityManager;
use League\Csv\Writer;

/**
 * A Writer class for Express Entry objects
 */
class CsvWriter
{

    /** @var Writer The writer we use to output */
    protected $writer;

    /**
     * @var Date
     */
    protected $dateFormatter;

    /**
     * @var string
     */
    private $datetime_format;

    /**
     * @var Service|null
     */
    private $siteService;

    /**
     * @var EntityManager
     */
    protected $entityManager;

    public function __construct(Writer $writer, Date $dateFormatter, EntityManager $entityManager, $datetime_format = DATE_ATOM)
    {
        $this->writer = $writer;
        $this->dateFormatter = $dateFormatter;
        $this->entityManager = $entityManager;
        $this->datetime_format = $datetime_format;
    }

    public function insertHeaders(Entity $entity)
    {
        $this->writer->insertOne(iterator_to_array($this->getHeaders($entity)));
    }

    /**
     * Insert all data from the passed EntryList
     * @param \Concrete\Core\Express\EntryList $list
     */
    public function insertEntryList(EntryList $list)
    {
        $list = clone $list;
        $this->writer->insertAll($this->projectList($list));
    }

    /**
     * A generator that takes an EntryList and converts it to CSV rows
     * @param \Concrete\Core\Express\EntryList $list
     * @return \Generator
     */
    private function projectList(EntryList $list)
    {
        $headers = array_keys(iterator_to_array($this->getHeaders($list->getEntity())));
        $statement = $list->deliverQueryObject()->execute();

        $total = 0;
        foreach ($statement as $result) {
            if ($entry = $list->getResult($result)) {
                yield $this->orderedEntry(iterator_to_array($this->projectEntry($entry)), $headers);
            }
            $total++;
            if ($total > 100) {
                $this->entityManager->clear();
            }
        }
    }

    /**
     * Return an entry in proper order
     * @param array $entry
     * @param array $headerKeys
     *
     * @return array
     */
    private function orderedEntry(array $entry, array $headerKeys)
    {
        $result = [];

        foreach ($headerKeys as $key) {
            $result[$key] = $entry[$key];
        }

        return $result;
    }

    /**
     * Turn an Entry into an array
     * @param \Concrete\Core\Entity\Express\Entry $entry
     * @return array
     */
    private function projectEntry(Entry $entry)
    {
        $date = $entry->getDateCreated();
{'fraction_non_alphanumeric': 0.1258827683615819, 'fraction_numerical': 0.0007062146892655367, 'mean_word_length': 2.523009950248756, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '973693', 'n_tokens_mistral': 1652, 'n_tokens_neox': 1611, 'n_words': 364}