<commit_before><?php

class loadDocumentsTask extends sfBaseTask {
  protected function configure() {
    $this->namespace = 'load';
    $this->name = 'Documents';
    $this->briefDescription = 'Load Documents data';
    $this->addOption('env', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'test');
    $this->addOption('app', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'frontend');
  }

  protected function execute($arguments = array(), $options = array()) {
    $dir = dirname(__FILE__).'/../../batch/documents/out/';
    $manager = new sfDatabaseManager($this->configuration);
    $doc = null;

    if (is_dir($dir)) {
      if ($dh = opendir($dir)) {
        while (($file = readdir($dh)) != false) {
          if ($file == ".." || $file == "." || $file == ".svn") continue;
          if ($doc) exit(1);
          foreach(file($dir.$file) as $line) {
            echo "$dir$file ... ";
            $json = json_decode($line);
            if (!$json) {
              echo "ERROR json : \n";
              continue;
            }
            if (!$json->source)
              {echo "ERROR source : \n"; continue;}
            if (!$json->legislature)
              {echo "ERROR legislature : \n"; continue;}
            if (!$json->id)
              {echo "ERROR id : \n"; continue;}
            if (!$json->numero)
              {echo "ERROR numero : \n"; continue;}
            if(!$json->date_depot)
              {echo "ERROR date_depot : \n"; continue;}
            if (!$json->dossier)
              {echo "ERROR dossier : \n"; continue;}
            if (!$json->type)
              {echo "ERROR type : \n"; continue;}
            $new = false;
            $doc = Doctrine::getTable('Texteloi')->find($json->id);
            if (!$doc) {
              $new = true;
              $doc = new Texteloi();
              $doc->id = $json->id;
              $doc->source = $json->source;
              $doc->legislature = $json->legislature;
              $doc->numero = $json->numero;
              if ($json->annexe != "") {
                if (Doctrine::getTable('Texteloi')->findByNumAnnexe($json->numero,$json->annexe))
                  {echo "ERROR numero + annexe déjà existant avec url différente : \n"; continue;}
                $doc->annexe = $json->annexe;
              }
              $doc->date = $json->date_depot;
              $doc->type = $json->type;
            }
            if ($json->date_publi)
              $doc->date = $json->date_publi;
            $doc->setDossier($json->dossier);
            if ($json->type_details)
              $doc->type_details = $json->type_details;
            if ($json->titre)
              $doc->titre = $json->titre;
            if ($json->categorie)
              $doc->categorie = $json->categorie;
            if ($json->auteurs)
              $doc->setAuteurs($json->auteurs);
            if ($json->motscles)<commit_msg>Parfois pas de date de publication<commit_after><?php

class loadDocumentsTask extends sfBaseTask {
  protected function configure() {
    $this->namespace = 'load';
    $this->name = 'Documents';
    $this->briefDescription = 'Load Documents data';
    $this->addOption('env', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'test');
    $this->addOption('app', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'frontend');
  }

  protected function execute($arguments = array(), $options = array()) {
    $dir = dirname(__FILE__).'/../../batch/documents/out/';
    $manager = new sfDatabaseManager($this->configuration);
    $doc = null;

    if (is_dir($dir)) {
      if ($dh = opendir($dir)) {
        while (($file = readdir($dh)) != false) {
          if ($file == ".." || $file == "." || $file == ".svn") continue;
          if ($doc) exit(1);
          foreach(file($dir.$file) as $line) {
            echo "$dir$file ... ";
            $json = json_decode($line);
            if (!$json) {
              echo "ERROR json : \n";
              continue;
            }
            if (!$json->source)
              {echo "ERROR source : \n"; continue;}
            if (!$json->legislature)
              {echo "ERROR legislature : \n"; continue;}
            if (!$json->id)
              {echo "ERROR id : \n"; continue;}
            if (!$json->numero)
              {echo "ERROR numero : \n"; continue;}
            if(!$json->date_depot)
              {echo "ERROR date_depot : \n"; continue;}
            if (!$json->dossier)
              {echo "ERROR dossier : \n"; continue;}
            if (!$json->type)
              {echo "ERROR type : \n"; continue;}
            $new = false;
            $doc = Doctrine::getTable('Texteloi')->find($json->id);
            if (!$doc) {
              $new = true;
              $doc = new Texteloi();
              $doc->id = $json->id;
              $doc->source = $json->source;
              $doc->legislature = $json->legislature;
              $doc->numero = $json->numero;
              if ($json->annexe != "") {
                if (Doctrine::getTable('Texteloi')->findByNumAnnexe($json->numero,$json->annexe))
                  {echo "ERROR numero + annexe déjà existant avec url différente : \n"; continue;}
                $doc->annexe = $json->annexe;
              }
              $doc->date = $json->date_depot;
              $doc->type = $json->type;
            }
            if (isset($json->date_publi) && $json->date_publi)
              $doc->date = $json->date_publi;
            $doc->setDossier($json->dossier);
            if ($json->type_details)
              $doc->type_details = $json->type_details;
            if ($json->titre)
              $doc->titre = $json->titre;
            if ($json->categorie)
              $doc->categorie = $json->categorie;
            if ($json->auteurs)
              $doc->setAuteurs($json->auteurs);
            if ($json->motscles)
{'fraction_non_alphanumeric': 0.16985526534686407, 'fraction_numerical': 0.00033272334054233906, 'mean_word_length': 1.7872044506258693, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '231851', 'n_tokens_mistral': 1751, 'n_tokens_neox': 1747, 'n_words': 395}