<commit_before><?php

class loadHemicyleTask extends sfBaseTask
{
  protected function configure()
  {
    $this->namespace = 'load';
    $this->name = 'Hemicycle';
    $this->briefDescription = 'Load Hemicycle data';
    $this->addOption('env', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'test');
    $this->addOption('app', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'frontend');
  }
 
  protected function execute($arguments = array(), $options = array())
  {
    // your code here
    $dir = dirname(__FILE__).'/../../batch/hemicycle/out/';
    $manager = new sfDatabaseManager($this->configuration);    

    if (is_dir($dir)) {
      if ($dh = opendir($dir)) {
        while (($file = readdir($dh)) !== false) {
	  $sections = array();
	  if (preg_match('/^\./', $file))
	    continue;
	  echo "$dir$file\n";
	  foreach(file($dir.$file) as $line) {
	    $json = json_decode($line);
	    if (!$json || !$json->intervention || !$json->date || !$json->heure || !$json->source) {
	      echo "ERROR json : ";
	      echo $line;
	      echo "\n";
	      continue;
	    }
	    $id = md5($json->intervention.$json->date.$json->heure.'hemicyle'.$json->context.$json->timestamp);
	    $intervention = Doctrine::getTable('Intervention')->findOneByMd5($id);
	    if(!$intervention) {
	      $intervention = new Intervention();
	      $intervention->md5 = $id;
	      $intervention->setIntervention($json->intervention);
	      if (preg_match('/question/i', $json->contexte))
    		$type = 'question';
	      else
        	$type = 'loi';
	      $intervention->date = $json->date;
	      $intervention->setSeance($type, $json->date, $json->heure, $json->session);
	      $intervention->setSource($json->source);
	      $intervention->setTimestamp($json->timestamp);
          if ($json->timestamp)
            $intervention->setContexte($json->contexte, $json->date.$json->heure, $json->timestamp);
	    }
	    if ($json->numeros_loi)
	      $intervention->setLois($json->numeros_loi);
	    if ($json->amendements)
	      $intervention->setAmendements($json->amendements);
	    if ($json->intervenant) {
	      $p = null;
	      if ($json->intervenant_url) {
            $p = doctrine::getTable('Parlementaire')
                  ->findOneByUrlAn($json->intervenant_url);
            if ($p) {
              $intervention->setParlementaire($p);
              $intervention->setFonction($json->fonction);
            }
	      }
	      if (!$p) {
            $intervention->setPersonnaliteByNom($json->intervenant, $json->fonction);
	      } else $p->free();
	    }<commit_msg>correction identification des questions orales pour exclure les questions sur un projet de loi<commit_after><?php

class loadHemicyleTask extends sfBaseTask
{
  protected function configure()
  {
    $this->namespace = 'load';
    $this->name = 'Hemicycle';
    $this->briefDescription = 'Load Hemicycle data';
    $this->addOption('env', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'test');
    $this->addOption('app', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'frontend');
  }
 
  protected function execute($arguments = array(), $options = array())
  {
    // your code here
    $dir = dirname(__FILE__).'/../../batch/hemicycle/out/';
    $manager = new sfDatabaseManager($this->configuration);    

    if (is_dir($dir)) {
      if ($dh = opendir($dir)) {
        while (($file = readdir($dh)) !== false) {
	  $sections = array();
	  if (preg_match('/^\./', $file))
	    continue;
	  echo "$dir$file\n";
	  foreach(file($dir.$file) as $line) {
	    $json = json_decode($line);
	    if (!$json || !$json->intervention || !$json->date || !$json->heure || !$json->source) {
	      echo "ERROR json : ";
	      echo $line;
	      echo "\n";
	      continue;
	    }
	    $id = md5($json->intervention.$json->date.$json->heure.'hemicyle'.$json->context.$json->timestamp);
	    $intervention = Doctrine::getTable('Intervention')->findOneByMd5($id);
	    if(!$intervention) {
	      $intervention = new Intervention();
	      $intervention->md5 = $id;
	      $intervention->setIntervention($json->intervention);
	      if (preg_match('/^question/i', $json->contexte))
    		$type = 'question';
	      else
        	$type = 'loi';
	      $intervention->date = $json->date;
	      $intervention->setSeance($type, $json->date, $json->heure, $json->session);
	      $intervention->setSource($json->source);
	      $intervention->setTimestamp($json->timestamp);
          if ($json->timestamp)
            $intervention->setContexte($json->contexte, $json->date.$json->heure, $json->timestamp);
	    }
	    if ($json->numeros_loi)
	      $intervention->setLois($json->numeros_loi);
	    if ($json->amendements)
	      $intervention->setAmendements($json->amendements);
	    if ($json->intervenant) {
	      $p = null;
	      if ($json->intervenant_url) {
            $p = doctrine::getTable('Parlementaire')
                  ->findOneByUrlAn($json->intervenant_url);
            if ($p) {
              $intervention->setParlementaire($p);
              $intervention->setFonction($json->fonction);
            }
	      }
	      if (!$p) {
            $intervention->setPersonnaliteByNom($json->intervenant, $json->fonction);
	      } else $p->free();
	    }
{'fraction_non_alphanumeric': 0.1785053644099149, 'fraction_numerical': 0.0011098779134295228, 'mean_word_length': 2.9011544011544013, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 109, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6184478', 'n_tokens_mistral': 1759, 'n_tokens_neox': 1657, 'n_words': 307}