<commit_before><?php

class importVracTask extends importAbstractTask
{

  const CSV_TRI = 0;
  const CSV_DOSSIER = 1;
  const CSV_CAMPAGNE = 2;
  const CSV_NUMERO_CONTRAT = 3;
  const CSV_DATE_ENREGISTREMENT = 4;
  const CSV_CODE_RECETTE_LOCALE = 5;
  const CSV_CODE_VITICULTEUR = 6;
  const CSV_CODE_CHAI_CAVE = 7;
  const CSV_CODE_NEGOCIANT = 8;
  const CSV_CODE_COURTIER = 9;
  const CSV_CODE_APPELLATION = 10;
  const CSV_TYPE_PRODUIT = 11;
  const CSV_MILLESIME = 12;
  const CSV_COTISATION_CVO_NEGOCIANT = 13;
  const CSV_COTISATION_CVO_VITICULTEUR = 14;
  const CSV_VOLUME = 15;
  const CSV_UNITE_VOLUME = 16;
  const CSV_COEF_CONVERSION_VOLUME = 17;
  const CSV_MODE_CONVERSION_VOLUME = 18;
  const CSV_VOLUME_PROPOSE_HL = 19;
  const CSV_VOLUME_ENLEVE_HL = 20;
  const CSV_PRIX_VENTE = 21;
  const CSV_CODE_DEVISE = 22;
  const CSV_UNITE_PRIX_VENTE = 23;
  const CSV_COEF_CONVERSION_PRIX = 24;
  const CSV_MODE_CONVERSION_PRIX = 25;
  const CSV_PRIX_AU_LITRE = 26;
  const CSV_CONTRAT_SOLDE = 27;
  const CSV_DATE_SIGNATURE_OU_CREATION = 28;
  const CSV_DATE_DERNIERE_MODIFICATION = 29;
  const CSV_CODE_SAISIE = 30;
  const CSV_DATE_LIVRAISON = 31;
  const CSV_CODE_MODE_PAIEMENT = 32;
  const CSV_COMPOSTAGE = 33;
  const CSV_TYPE_CONTRAT = 34;
  const CSV_ATTENTE_ORIGINAL = 35;
  const CSV_CATEGORIE_VIN = 36;
  const CSV_CEPAGE = 37;
  const CSV_MILLESIME_ANNEE = 38;
  const CSV_PRIX_HORS_CVO = 39;
  const CSV_PRIX_CVO_INCLUSE = 40;
  const CSV_TAUX_CVO_GLOBAL = 41;
  const CSV_INDICATEUR_CONTRAT_PRIX_VARIABLE = 42;
  const CSV_PRIX_DEFINITIF = 43;
  const CSV_INDICATEUR_PRIX_DEFINITIF = 44;

  const CSV_TYPE_PRODUIT_INDETERMINE = 0;
  const CSV_TYPE_PRODUIT_RAISINS = 1;
  const CSV_TYPE_PRODUIT_MOUTS = 2;
  const CSV_TYPE_PRODUIT_VIN_VRAC = 3;
  const CSV_TYPE_PRODUIT_TIRE_BOUCHE = 5;
  const CSV_TYPE_PRODUIT_VIN_LATTES = 6;
  const CSV_TYPE_PRODUIT_VIN_CRD = 7;
  const CSV_TYPE_PRODUIT_VIN_BOUTEILLE = 8;

  const CSV_UNITE_VOLUME_HL = 'hl';
  const CSV_UNITE_VOLUME_L = 'l';
  const CSV_UNITE_VOLUME_KG = 'kg';
  const CSV_UNITE_VOLUME_BARRIQUE = 'ba';

  const CSV_TYPE_CONTRAT_SORTIES_COOP = 'C';
  const CSV_TYPE_CONTRAT_PLURIANNUEL_PRIX_A_DEFINIR = 'D';
  const CSV_TYPE_CONTRAT_FITRANEG = 'F';
  const CSV_TYPE_CONTRAT_FERMAGE = 'M';
  const CSV_TYPE_CONTRAT_PLURIANNUEL = 'P';
  const CSV_TYPE_CONTRAT_QUINQUENNAL = 'Q';
  const CSV_TYPE_CONTRAT_PAS_TRANSACTION_FINANCIERE = 'T';
  const CSV_TYPE_CONTRAT_VINAIGRERIE = 'V';

  protected function configure()
  {
    // // add your own arguments here
    $this->addArguments(array(
       new sfCommandArgument('file', sfCommandArgument::REQUIRED, "Fichier csv pour l'import"),
    ));

    $this->addOptions(array(
      new sfCommandOption('application', null, sfCommandOption::PARAMETER_REQUIRED, 'The application name', 'vinsdeloire'),
      new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'dev'),
      new sfCommandOption('connection', null, sfCommandOption::PARAMETER_REQUIRED, 'The connection name', 'default'),
      // add your own options here
    ));

    $this->namespace        = 'import';
    $this->name             = 'vrac';
    $this->briefDescription = '';
    $this->detailedDescription = <<<EOF
The [importVrac|INFO] task does things.
Call it with:

  [php symfony importEtablissement|INFO]
EOF;
  }

  protected function execute($arguments = array(), $options = array())
  {
    // initialize the database connection
    $databaseManager = new sfDatabaseManager($this->configuration);
    $connection = $databaseManager->getDatabase($options['connection'])->getConnection();

    set_time_limit(0);
    $i = 1;
    foreach(file($arguments['file']) as $line) {
    	$data = str_getcsv($line, ';');

      try{
        $vrac = $this->importVrac($data);
      } catch (Exception $e) {
        $this->log(sprintf("%s (ligne %s) : %s", $e->getMessage(), $i, implode($data, ";")));

        continue;
      }
      $vrac->save();

      $i++;
    }

  }

  public function importVrac($line) {

        $type_transaction = $this->convertTypeTransaction($line);   

        if (!$type_transaction) {
         
         throw new sfException(sprintf("Le type de transaction est inexistant", $type_transaction));
        }
        
        $v = VracClient::getInstance()->findByNumContrat($this->constructNumeroContrat($line));
        
        if (!$v) {
          $v = new Vrac();
          $v->numero_contrat = $this->constructNumeroContrat($line);
        }

        $v->numero_archive = $this->getNumeroArchive($line);

        $v->label = array();

        $date = $this->getDateCampagne($line);
        $v->date_signature = $date->format('Y-m-d');
        $v->date_campagne = $date->format('Y-m-d');
        $v->valide->date_saisie = $this->convertToDateObject($line[self::CSV_DATE_ENREGISTREMENT])->format('Y-m-d');

        $v->vendeur_identifiant = $this->getIdentifiantVendeur($line);
        $v->acheteur_identifiant = $this->getIdentifiantAcheteur($line);
        $v->mandataire_identifiant = null;

        if ($line[self::CSV_CODE_COURTIER]) {
          $v->mandataire_identifiant = $this->getIdentifiantCourtier($line);
        }

        $v->produit = $this->getHash($line[self::CSV_CODE_APPELLATION]);

        $v->millesime = $line[self::CSV_MILLESIME_ANNEE] ? (int)$line[self::CSV_MILLESIME_ANNEE] : null;

        if (!$v->getVendeurObject()) {
          
          throw new sfException(sprintf("L'etablissement %s n'existe pas", $v->vendeur_identifiant));
        }

        if (!$v->getAcheteurObject()) {
          
          throw new sfException(sprintf("L'etablissement %s n'existe pas", $v->acheteur_identifiant));
        }

        if (!$v->mandataire_identifiant || !$v->getMandataireObject()) {
          $v->mandataire_identifiant = null;
        } else {
          $v->mandataire_exist = 1;
        }

        $v->type_transaction = $type_transaction;
        
        if (in_array($v->type_transaction, array(VracClient::TYPE_TRANSACTION_VIN_BOUTEILLE))) {
          	$v->bouteilles_contenance_volume = $line[self::CSV_COEF_CONVERSION_PRIX] * 0.01;
		$v->bouteilles_contenance_libelle = $this->getBouteilleContenanceLibelle($v->bouteilles_contenance_volume);
          	$v->bouteilles_quantite = (int)round($this->convertToFloat($line[self::CSV_VOLUME_PROPOSE_HL]) / $v->bouteilles_contenance_volume);
        } elseif(in_array($v->type_transaction, array(VracClient::TYPE_TRANSACTION_MOUTS,
                                                      VracClient::TYPE_TRANSACTION_VIN_VRAC))) {
          	$v->jus_quantite = $this->convertToFloat($line[self::CSV_VOLUME_PROPOSE_HL]);
        } elseif(in_array($v->type_transaction, array(VracClient::TYPE_TRANSACTION_RAISINS))) {
          	$v->raisin_quantite = round($this->convertToFloat($line[self::CSV_VOLUME_PROPOSE_HL] * $this->getDensite($line) * 100), 2);
        }

        $v->volume_propose = $this->convertToFloat($line[self::CSV_VOLUME_PROPOSE_HL]);

        $v->volume_enleve = $this->convertToFloat($line[self::CSV_VOLUME_ENLEVE_HL]);

        $v->prix_initial_unitaire = round($this->convertToFloat($this->calculPrixInitialUnitaire($v, $line)), 2);
        $v->prix_initial_unitaire_hl = round($this->convertToFloat($line[self::CSV_PRIX_AU_LITRE]), 2);
        $v->prix_initial_total = $v->prix_initial_unitaire_hl * $v->volume_propose;

        $v->type_contrat = $this->convertTypeContrat($line[self::CSV_TYPE_CONTRAT]);

        $v->prix_variable = $this->convertOuiNon($line[self::CSV_INDICATEUR_PRIX_DEFINITIF]);

        if(!$v->hasPrixVariable()) {
          $v->prix_unitaire = $v->prix_initial_unitaire;
          $v->prix_unitaire_hl = $v->prix_initial_unitaire_hl;
          $v->prix_total = $v->prix_initial_total;
        }

        if($v->hasPrixVariable() && $line[self::CSV_PRIX_DEFINITIF]) {
          $v->prix_unitaire = round($this->convertToFloat($this->calculPrixDefinitifUnitaire($v, $line)), 2);
          $v->prix_unitaire_hl = round($this->convertToFloat($line[self::CSV_PRIX_DEFINITIF]), 2);
          $v->prix_total = $v->prix_unitaire_hl * $v->volume_propose;
        }

        if(!round($this->convertToFloat($line[self::CSV_COTISATION_CVO_VITICULTEUR]), 2)) {
          $v->cvo_repartition = VracClient::CVO_REPARTITION_0_VINAIGRERIE;
        } elseif(abs($line[self::CSV_COTISATION_CVO_VITICULTEUR] - $line[self::CSV_COTISATION_CVO_NEGOCIANT]) <= 0.01) {
          $v->cvo_repartition = VracClient::CVO_REPARTITION_50_50;
        } else {
          $v->cvo_repartition = VracClient::CVO_REPARTITION_100_VITI;          
        }

        if($v->cvo_repartition == VracClient::CVO_REPARTITION_100_VITI && $line[self::CSV_COTISATION_CVO_NEGOCIANT] > 0) {
          
          $this->logLigne('WARNING', sprintf("Incohérence de CVO VITI v:%s n:%s", $line[self::CSV_COTISATION_CVO_VITICULTEUR], $line[self::CSV_COTISATION_CVO_NEGOCIANT]), $line);
        }

        if($v->cvo_repartition == VracClient::CVO_REPARTITION_0_VINAIGRERIE && $line[self::CSV_COTISATION_CVO_NEGOCIANT]) {

          $this->logLigne('WARNING', sprintf("Incohérence de CVO VINAIG v:%s n:%s", $line[self::CSV_COTISATION_CVO_VITICULTEUR], $line[self::CSV_COTISATION_CVO_NEGOCIANT]), $line);
        }

        $v->attente_original = $this->convertOuiNon($line[self::CSV_ATTENTE_ORIGINAL]);

        $v->categorie_vin = $line[self::CSV_CATEGORIE_VIN] == "D" ? VracClient::CATEGORIE_VIN_DOMAINE : VracClient::CATEGORIE_VIN_GENERIQUE;
<commit_msg>Import VRAC: - Si kg on set l'unité en kg<commit_after><?php

class importVracTask extends importAbstractTask
{

  const CSV_TRI = 0;
  const CSV_DOSSIER = 1;
  const CSV_CAMPAGNE = 2;
  const CSV_NUMERO_CONTRAT = 3;
  const CSV_DATE_ENREGISTREMENT = 4;
  const CSV_CODE_RECETTE_LOCALE = 5;
  const CSV_CODE_VITICULTEUR = 6;
  const CSV_CODE_CHAI_CAVE = 7;
  const CSV_CODE_NEGOCIANT = 8;
  const CSV_CODE_COURTIER = 9;
  const CSV_CODE_APPELLATION = 10;
  const CSV_TYPE_PRODUIT = 11;
  const CSV_MILLESIME = 12;
  const CSV_COTISATION_CVO_NEGOCIANT = 13;
  const CSV_COTISATION_CVO_VITICULTEUR = 14;
  const CSV_VOLUME = 15;
  const CSV_UNITE_VOLUME = 16;
  const CSV_COEF_CONVERSION_VOLUME = 17;
  const CSV_MODE_CONVERSION_VOLUME = 18;
  const CSV_VOLUME_PROPOSE_HL = 19;
  const CSV_VOLUME_ENLEVE_HL = 20;
  const CSV_PRIX_VENTE = 21;
  const CSV_CODE_DEVISE = 22;
  const CSV_UNITE_PRIX_VENTE = 23;
  const CSV_COEF_CONVERSION_PRIX = 24;
  const CSV_MODE_CONVERSION_PRIX = 25;
  const CSV_PRIX_AU_LITRE = 26;
  const CSV_CONTRAT_SOLDE = 27;
  const CSV_DATE_SIGNATURE_OU_CREATION = 28;
  const CSV_DATE_DERNIERE_MODIFICATION = 29;
  const CSV_CODE_SAISIE = 30;
  const CSV_DATE_LIVRAISON = 31;
  const CSV_CODE_MODE_PAIEMENT = 32;
  const CSV_COMPOSTAGE = 33;
  const CSV_TYPE_CONTRAT = 34;
  const CSV_ATTENTE_ORIGINAL = 35;
  const CSV_CATEGORIE_VIN = 36;
  const CSV_CEPAGE = 37;
  const CSV_MILLESIME_ANNEE = 38;
  const CSV_PRIX_HORS_CVO = 39;
  const CSV_PRIX_CVO_INCLUSE = 40;
  const CSV_TAUX_CVO_GLOBAL = 41;
  const CSV_INDICATEUR_CONTRAT_PRIX_VARIABLE = 42;
  const CSV_PRIX_DEFINITIF = 43;
  const CSV_INDICATEUR_PRIX_DEFINITIF = 44;

  const CSV_TYPE_PRODUIT_INDETERMINE = 0;
  const CSV_TYPE_PRODUIT_RAISINS = 1;
  const CSV_TYPE_PRODUIT_MOUTS = 2;
  const CSV_TYPE_PRODUIT_VIN_VRAC = 3;
  const CSV_TYPE_PRODUIT_TIRE_BOUCHE = 5;
  const CSV_TYPE_PRODUIT_VIN_LATTES = 6;
  const CSV_TYPE_PRODUIT_VIN_CRD = 7;
  const CSV_TYPE_PRODUIT_VIN_BOUTEILLE = 8;

  const CSV_UNITE_VOLUME_HL = 'hl';
  const CSV_UNITE_VOLUME_L = 'l';
  const CSV_UNITE_VOLUME_KG = 'kg';
  const CSV_UNITE_VOLUME_BARRIQUE = 'ba';

  const CSV_TYPE_CONTRAT_SORTIES_COOP = 'C';
  const CSV_TYPE_CONTRAT_PLURIANNUEL_PRIX_A_DEFINIR = 'D';
  const CSV_TYPE_CONTRAT_FITRANEG = 'F';
  const CSV_TYPE_CONTRAT_FERMAGE = 'M';
  const CSV_TYPE_CONTRAT_PLURIANNUEL = 'P';
  const CSV_TYPE_CONTRAT_QUINQUENNAL = 'Q';
  const CSV_TYPE_CONTRAT_PAS_TRANSACTION_FINANCIERE = 'T';
  const CSV_TYPE_CONTRAT_VINAIGRERIE = 'V';

  protected function configure()
  {
    // // add your own arguments here
    $this->addArguments(array(
       new sfCommandArgument('file', sfCommandArgument::REQUIRED, "Fichier csv pour l'import"),
    ));

    $this->addOptions(array(
      new sfCommandOption('application', null, sfCommandOption::PARAMETER_REQUIRED, 'The application name', 'vinsdeloire'),
      new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'dev'),
      new sfCommandOption('connection', null, sfCommandOption::PARAMETER_REQUIRED, 'The connection name', 'default'),
      // add your own options here
    ));

    $this->namespace        = 'import';
    $this->name             = 'vrac';
    $this->briefDescription = '';
    $this->detailedDescription = <<<EOF
The [importVrac|INFO] task does things.
Call it with:

  [php symfony importEtablissement|INFO]
EOF;
  }

  protected function execute($arguments = array(), $options = array())
  {
    // initialize the database connection
    $databaseManager = new sfDatabaseManager($this->configuration);
    $connection = $databaseManager->getDatabase($options['connection'])->getConnection();

    set_time_limit(0);
    $i = 1;
    foreach(file($arguments['file']) as $line) {
    	$data = str_getcsv($line, ';');

      try{
        $vrac = $this->importVrac($data);
      } catch (Exception $e) {
        $this->log(sprintf("%s (ligne %s) : %s", $e->getMessage(), $i, implode($data, ";")));

        continue;
      }
      $vrac->save();

      $i++;
    }

  }

  public function importVrac($line) {

        $type_transaction = $this->convertTypeTransaction($line);   

        if (!$type_transaction) {
         
         throw new sfException(sprintf("Le type de transaction est inexistant", $type_transaction));
        }
        
        $v = VracClient::getInstance()->findByNumContrat($this->constructNumeroContrat($line));
        
        if (!$v) {
          $v = new Vrac();
          $v->numero_contrat = $this->constructNumeroContrat($line);
        }

        $v->numero_archive = $this->getNumeroArchive($line);

        $v->label = array();

        $date = $this->getDateCampagne($line);
        $v->date_signature = $date->format('Y-m-d');
        $v->date_campagne = $date->format('Y-m-d');
        $v->valide->date_saisie = $this->convertToDateObject($line[self::CSV_DATE_ENREGISTREMENT])->format('Y-m-d');

        $v->vendeur_identifiant = $this->getIdentifiantVendeur($line);
        $v->acheteur_identifiant = $this->getIdentifiantAcheteur($line);
        $v->mandataire_identifiant = null;

        if ($line[self::CSV_CODE_COURTIER]) {
          $v->mandataire_identifiant = $this->getIdentifiantCourtier($line);
        }

        $v->produit = $this->getHash($line[self::CSV_CODE_APPELLATION]);

        $v->millesime = $line[self::CSV_MILLESIME_ANNEE] ? (int)$line[self::CSV_MILLESIME_ANNEE] : null;

        if (!$v->getVendeurObject()) {
          
          throw new sfException(sprintf("L'etablissement %s n'existe pas", $v->vendeur_identifiant));
        }

        if (!$v->getAcheteurObject()) {
          
          throw new sfException(sprintf("L'etablissement %s n'existe pas", $v->acheteur_identifiant));
        }

        if (!$v->mandataire_identifiant || !$v->getMandataireObject()) {
          $v->mandataire_identifiant = null;
        } else {
          $v->mandataire_exist = 1;
        }

        $v->type_transaction = $type_transaction;
        
        if (in_array($v->type_transaction, array(VracClient::TYPE_TRANSACTION_VIN_BOUTEILLE))) {
          	$v->bouteilles_contenance_volume = $line[self::CSV_COEF_CONVERSION_PRIX] * 0.01;
		$v->bouteilles_contenance_libelle = $this->getBouteilleContenanceLibelle($v->bouteilles_contenance_volume);
          	$v->bouteilles_quantite = (int)round($this->convertToFloat($line[self::CSV_VOLUME_PROPOSE_HL]) / $v->bouteilles_contenance_volume);
        } elseif(in_array($v->type_transaction, array(VracClient::TYPE_TRANSACTION_MOUTS,
                                                      VracClient::TYPE_TRANSACTION_VIN_VRAC))) {
          	$v->jus_quantite = $this->convertToFloat($line[self::CSV_VOLUME_PROPOSE_HL]);
        } elseif(in_array($v->type_transaction, array(VracClient::TYPE_TRANSACTION_RAISINS))) {
		if($line[self::CSV_UNITE_VOLUME] == 'kg') {  
			$v->raisin_quantite = round($this->convertToFloat($line[self::CSV_VOLUME], 2));
		} else {
			$v->raisin_quantite = round($this->convertToFloat($line[self::CSV_VOLUME_PROPOSE_HL] * $this->getDensite($line) * 100), 2);
		}	
	}

        $v->volume_propose = $this->convertToFloat($line[self::CSV_VOLUME_PROPOSE_HL]);

        $v->volume_enleve = $this->convertToFloat($line[self::CSV_VOLUME_ENLEVE_HL]);

        $v->prix_initial_unitaire = round($this->convertToFloat($this->calculPrixInitialUnitaire($v, $line)), 2);
        $v->prix_initial_unitaire_hl = round($this->convertToFloat($line[self::CSV_PRIX_AU_LITRE]), 2);
        $v->prix_initial_total = round($v->prix_initial_unitaire_hl * $v->volume_propose, 2);

        $v->type_contrat = $this->convertTypeContrat($line[self::CSV_TYPE_CONTRAT]);

        $v->prix_variable = $this->convertOuiNon($line[self::CSV_INDICATEUR_PRIX_DEFINITIF]);

        if(!$v->hasPrixVariable()) {
          $v->prix_unitaire = $v->prix_initial_unitaire;
          $v->prix_unitaire_hl = $v->prix_initial_unitaire_hl;
          $v->prix_total = $v->prix_initial_total;
        }

        if($v->hasPrixVariable() && $line[self::CSV_PRIX_DEFINITIF]) {
          $v->prix_unitaire = round($this->convertToFloat($this->calculPrixDefinitifUnitaire($v, $line)), 2);
          $v->prix_unitaire_hl = round($this->convertToFloat($line[self::CSV_PRIX_DEFINITIF]), 2);
          $v->prix_total = round($v->prix_unitaire_hl * $v->volume_propose, 2);;
        }

        if(!round($this->convertToFloat($line[self::CSV_COTISATION_CVO_VITICULTEUR]), 2)) {
          $v->cvo_repartition = VracClient::CVO_REPARTITION_0_VINAIGRERIE;
        } elseif(abs($line[self::CSV_COTISATION_CVO_VITICULTEUR] - $line[self::CSV_COTISATION_CVO_NEGOCIANT]) <= 0.01) {
          $v->cvo_repartition = VracClient::CVO_REPARTITION_50_50;
        } else {
          $v->cvo_repartition = VracClient::CVO_REPARTITION_100_VITI;          
        }

        if($v->cvo_repartition == VracClient::CVO_REPARTITION_100_VITI && $line[self::CSV_COTISATION_CVO_NEGOCIANT] > 0) {
          
          $this->logLigne('WARNING', sprintf("Incohérence de CVO VITI v:%s n:%s", $line[self::CSV_COTISATION_CVO_VITICULTEUR], $line[self::CSV_COTISATION_CVO_NEGOCIANT]), $line);
        }

        if($v->cvo_repartition == VracClient::CVO_REPARTITION_0_VINAIGRERIE && $line[self::CSV_COTISATION_CVO_NEGOCIANT]) {

          $this->logLigne('WARNING', sprintf("Incohérence de CVO VINAIG v:%s n:%s", $line[self::CSV_COTISATION_CVO_VITICULTEUR], $line[self::CSV_COTISATION_CVO_NEGOCIANT]), $line);
        }

        $v->attente_original = $this->convertOuiNon($line[self::CSV_ATTENTE_ORIGINAL]);

        $v->categorie_vin = $line[self::CSV_CATEGORIE_VIN] == "D" ? VracClient::CATEGORIE_VIN_DOMAINE : VracClient::CATEGORIE_VIN_GENERIQUE;

{'fraction_non_alphanumeric': 0.1337060452479231, 'fraction_numerical': 0.012592089450859502, 'mean_word_length': 3.8578680203045685, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 221, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2013631', 'n_tokens_mistral': 7663, 'n_tokens_neox': 7239, 'n_words': 996}