<commit_before><?php

class DeclarationsExportCsvTask extends sfBaseTask
{

    protected function configure()
    {
        $this->addArguments(array(
            new sfCommandArgument('type', sfCommandArgument::REQUIRED, "Type du document"),
            new sfCommandArgument('campagne', sfCommandArgument::REQUIRED, "Campagne"),
            new sfCommandArgument('validation', sfCommandArgument::OPTIONAL, "Document validé uniquement", true),
        ));

        $this->addOptions(array(
            new sfCommandOption('application', null, sfCommandOption::PARAMETER_REQUIRED, 'The application name', 'declaration'),
            new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'prod'),
            new sfCommandOption('connection', null, sfCommandOption::PARAMETER_REQUIRED, 'The connection name', 'default'),
            new sfCommandOption('header', null, sfCommandOption::PARAMETER_REQUIRED, 'Add header in CSV', true),
            new sfCommandOption('sleep_second', null, sfCommandOption::PARAMETER_REQUIRED, 'secont to wait', false),
            new sfCommandOption('sleep_step', null, sfCommandOption::PARAMETER_REQUIRED, 'nb doc before wait', false),
            new sfCommandOption('region', null, sfCommandOption::PARAMETER_REQUIRED, "region de l'ODG (si non renseignée toutes les régions sont utilisées)", null),
        ));

        $this->namespace = 'declarations';
        $this->name = 'export-csv';
        $this->briefDescription = "Export CSV d'une declaration";
        $this->detailedDescription = <<<EOF
EOF;
    }

    protected function execute($arguments = array(), $options = array())
    {
        // initialize the database connection
        $databaseManager = new sfDatabaseManager($this->configuration);
        $connection = $databaseManager->getDatabase($options['connection'])->getConnection();

        if($options["header"]) {
            $className = DeclarationClient::getInstance()->getExportCsvClassName($arguments['type']);
            echo $className::getHeaderCsv();
        }

        $ids = DeclarationClient::getInstance()->getIds($arguments['type'], $arguments['campagne']);

        $sleepSecond = false;
        if($options['sleep_second']) {
            $sleepSecond = $options['sleep_second']*1;
        }

        $sleepStep = false;
        if($options['sleep_step']) {
            $sleepStep = $options['sleep_step']*1;
        }

        $step = 0;

        $region = null;
        if($options["region"]) {
            $region = $options['region'];
        }

        foreach($ids as $id) {
            $doc = null;
            try{
              $doc = DeclarationClient::getInstance()->find($id);
            }catch(sfException $e){
              continue;
            }
            $export = DeclarationClient::getInstance()->getExportCsvObject($doc, false, $region);

            if($arguments['validation'] && $doc->exist('validation') && !$doc->validation) {
                continue;
            }

            if(method_exists($doc, "isExcluExportCsv") && $doc->isExcluExportCsv()) {
                continue;
            }

            echo $export->export();
            $step++;
            if($sleepStep && $sleepSecond && $step > $sleepStep) {
                sleep($sleepSecond);
                $step = 0;
            }
        }


    }
}<commit_msg>Protège contre les "no data to parse" liés à de trop nombreuses requetes<commit_after><?php

class DeclarationsExportCsvTask extends sfBaseTask
{

    protected function configure()
    {
        $this->addArguments(array(
            new sfCommandArgument('type', sfCommandArgument::REQUIRED, "Type du document"),
            new sfCommandArgument('campagne', sfCommandArgument::REQUIRED, "Campagne"),
            new sfCommandArgument('validation', sfCommandArgument::OPTIONAL, "Document validé uniquement", true),
        ));

        $this->addOptions(array(
            new sfCommandOption('application', null, sfCommandOption::PARAMETER_REQUIRED, 'The application name', 'declaration'),
            new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'prod'),
            new sfCommandOption('connection', null, sfCommandOption::PARAMETER_REQUIRED, 'The connection name', 'default'),
            new sfCommandOption('header', null, sfCommandOption::PARAMETER_REQUIRED, 'Add header in CSV', true),
            new sfCommandOption('sleep_second', null, sfCommandOption::PARAMETER_REQUIRED, 'secont to wait', false),
            new sfCommandOption('sleep_step', null, sfCommandOption::PARAMETER_REQUIRED, 'nb doc before wait', 1000),
            new sfCommandOption('region', null, sfCommandOption::PARAMETER_REQUIRED, "region de l'ODG (si non renseignée toutes les régions sont utilisées)", null),
        ));

        $this->namespace = 'declarations';
        $this->name = 'export-csv';
        $this->briefDescription = "Export CSV d'une declaration";
        $this->detailedDescription = <<<EOF
EOF;
    }

    protected function execute($arguments = array(), $options = array())
    {
        // initialize the database connection
        $databaseManager = new sfDatabaseManager($this->configuration);
        $connection = $databaseManager->getDatabase($options['connection'])->getConnection();

        if($options["header"]) {
            $className = DeclarationClient::getInstance()->getExportCsvClassName($arguments['type']);
            echo $className::getHeaderCsv();
        }

        $ids = DeclarationClient::getInstance()->getIds($arguments['type'], $arguments['campagne']);

        $sleepSecond = false;
        if($options['sleep_second']) {
            $sleepSecond = $options['sleep_second']*1;
        }

        $sleepStep = $options['sleep_step']*1;

        $step = 0;

        $region = null;
        if($options["region"]) {
            $region = $options['region'];
        }

        foreach($ids as $id) {
            $tobeexported = true;
            while ($tobeexported) {
                try {
                    $doc = null;
                    try{
                      $doc = DeclarationClient::getInstance()->find($id);
                    }catch(sfException $e){
                      continue 2;
                    }
                    $export = DeclarationClient::getInstance()->getExportCsvObject($doc, false, $region);

                    if($arguments['validation'] && $doc->exist('validation') && !$doc->validation) {
                        $tobeexported = false;
                        continue 2;
                    }

                    if(method_exists($doc, "isExcluExportCsv") && $doc->isExcluExportCsv()) {
                        $tobeexported = false;
                        continue 2;
                    }

                    echo $export->export();

                }catch(InvalidArgumentException $e) {
                    sleep(60);
                    continue;
                }
                $tobeexported = false;
            }
            $step++;
            if($sleepStep && $sleepSecond && $step > $sleepStep) {
                sleep($sleepSecond);
                $step = 0;
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.13218550402665927, 'fraction_numerical': 0.002360455429047487, 'mean_word_length': 2.3393602225312935, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2772673', 'n_tokens_mistral': 1848, 'n_tokens_neox': 1814, 'n_words': 381}