<commit_before><?php

class updateDeputesTask extends sfBaseTask
{
  protected function configure()
  {
    $this->namespace = 'update';
    $this->name = 'Deputes';
    $this->briefDescription = 'Update Deputes';
    $this->addOption('env', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'test');
    $this->addOption('application', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'frontend');
  }

  protected function splitArrayJson($json) {
    $res = array();
    foreach($json as $j) {
      if ($j)
        array_push($res, explode(' / ', $j));
    }
    return $res;
  }

  protected function execute($arguments = array(), $options = array())
  {
    $dir = dirname(__FILE__).'/../../batch/depute/json/';
    $manager = new sfDatabaseManager($this->configuration);

    if (sfConfig::get('app_legislature') == 13)
      $villes = json_decode(file_get_contents($dir.'../static/villes_2007.json'));
    else $villes = json_decode(file_get_contents($dir.'../static/villes_2012.json'));

    $sites = array();
    $row = 0;
    if (($handle = fopen($dir."../twitter.csv", "r")) !== FALSE) {
      fgetcsv($handle);
      while (($data = fgetcsv($handle)) !== FALSE) {
        $row++;
        if ($row == 1) continue;
        $sites[$data[19]] = explode("|", $data[17]);
        $sites[$data[19]][] = "https://twitter.com/".$data[0];
      }
      fclose($handle);
    }

    if (is_dir($dir)) {
      if ($dh = opendir($dir)) {
        while (($file = readdir($dh)) !== false) {
          if (preg_match('/^\./', $file))
            continue;
    //    echo "$dir$file\n";
          foreach(file($dir.$file) as $line) {
            $json = json_decode($line);
            if (!isset($json->nom) || !strlen($json->nom)) {
              echo "WARNING: ".$dir.$file." doesn't appear to be a json file\n";
              continue;
            }
            $json->nom = trim($json->nom);
            $parl = Doctrine::getTable('Parlementaire')->findOneByNom($json->nom);
            if (!$parl) {
              echo "[INFO] Nouveau député : ".$json->nom." ".$json->url_institution."\n";
              $parl = new Parlementaire();
              $parl->type = 'depute';
              $parl->nom = $json->nom;
              $parl->nom_de_famille = $json->nom_de_famille;
            }
            $parl->sexe = $json->sexe;
            if ($json->date_naissance)
              $parl->date_naissance = $json->date_naissance;
            if ($json->lieu_naissance)
              $parl->lieu_naissance = $json->lieu_naissance;
            if ($json->circonscription)
              $parl->circonscription = $json->circonscription;
            else {
              if ($json->departement)
                $parl->setDepartementParNumero($json->departement);
              if ($json->num_circonscription)
                $parl->num_circo = $json->num_circonscription;
            }
            if ($json->adresses)
              $parl->adresses = $json->adresses;
            if ($json->premiers_mandats)
              $parl->anciens_mandats = $json->premiers_mandats;
            if ($json->autresmandats)
              $parl->autres_mandats = $json->autresmandats;
            if ($json->anciensmandats)
              $parl->anciens_autres_mandats = $json->anciensmandats;
            $parl->debut_mandat = $json->debut_mandat;
            $parl->fin_mandat = $json->fin_mandat;
            if ($json->groupe && (!$parl->fin_mandat || $parl->fin_mandat < $parl->debut_mandat))
              $parl->groupe = $this->splitArrayJson($json->groupe);
            if ($json->parti)
              $parl->parti = $json->parti;
            if ($json->fonctions)
              $parl->fonctions = $this->splitArrayJson($json->fonctions);
            $parl->extras = ($json->extras ? $this->splitArrayJson($json->extras) : array());
            if ($json->groupes)
              $parl->groupes = $this->splitArrayJson($json->groupes);
            if ($json->id_institution)
              $parl->id_an = $json->id_institution;
            if ($json->mails)
              $parl->mails = $json->mails;
            if ($json->photo)
              $parl->photo = $json->photo;
            if ($json->place_hemicycle)
              $parl->place_hemicycle = $json->place_hemicycle;
            if ($json->profession)
              $parl->profession = $json->profession;
            $done_sites = array();
            if ($json->sites_web)
              foreach (array_keys($json->sites_web) as $i) {
                $json->sites_web[$i] = preg_replace("|(://[^/]+)/$|", "$1", trim($json->sites_web[$i]));
                $rootsite = strtolower(preg_replace("#^(https?://|www\.|m\.|fr\.|fr-fr\.)*(.*?)[\s/]*$#", "$2", $json->sites_web[$i]));
                if (!isset($done_sites[$rootsite]))
                  $done_sites[$rootsite] = 1;
                else unset($json->sites_web[$i]);
              }
            if (isset($sites[$parl->slug]) && $sites[$parl->slug])
              foreach ($sites[$parl->slug] as $site) {
                $site = preg_replace("|(://[^/]+)/$|", "$1", trim($site));
                $rootsite = strtolower(preg_replace("#^(https?://|www\.|m\.|fr\.|fr-fr\.)*(.*?)[\s/]*$#", "$2", $site));
                if (!isset($done_sites[$rootsite]))
                  $json->sites_web[] = $site;
                $done_sites[$rootsite] = 1;
              }
            if ($json->sites_web)
              $parl->sites_web = $json->sites_web;
            else if ($parl->sites_web && !preg_match('/^a:/', $parl->sites_web))
              $parl->sites_web = array($parl->sites_web);
            $parl->collaborateurs = $json->collabs;
            if ($json->url_institution)
              $parl->url_an = $json->url_institution;
            if ($json->suppleant_de)
              $parl->setSuppleantDe($json->suppleant_de);
            if ($json->url_ancien_cpc)
              $parl->url_ancien_cpc = $json->url_ancien_cpc;
            if ($json->url_nouveau_cpc)
              $parl->url_nouveau_cpc = $json->url_nouveau_cpc;
            $parl->villes = $villes->{$parl->getNumDepartement()}->{$parl->num_circo};
            $parl->save();
            if ($json->suppleant) {
              $suppl = Doctrine::getTable('Parlementaire')->findOneByNom(preg_replace('/^\s*M[.mle]* /', '', $json->suppleant));
              if ($suppl) {
                $suppl->setSuppleantDe($parl->nom);
                $suppl->save();
              }
            }
          }
        }
      }
    }
  }
}
<commit_msg>don't add twitter account from AN if we already have one (new ones from AN will be loaded directly into it)<commit_after><?php

class updateDeputesTask extends sfBaseTask
{
  protected function configure()
  {
    $this->namespace = 'update';
    $this->name = 'Deputes';
    $this->briefDescription = 'Update Deputes';
    $this->addOption('env', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'test');
    $this->addOption('application', null, sfCommandOption::PARAMETER_OPTIONAL, 'Changes the environment this task is run in', 'frontend');
  }

  protected function splitArrayJson($json) {
    $res = array();
    foreach($json as $j) {
      if ($j)
        array_push($res, explode(' / ', $j));
    }
    return $res;
  }

  protected function execute($arguments = array(), $options = array())
  {
    $dir = dirname(__FILE__).'/../../batch/depute/json/';
    $manager = new sfDatabaseManager($this->configuration);

    if (sfConfig::get('app_legislature') == 13)
      $villes = json_decode(file_get_contents($dir.'../static/villes_2007.json'));
    else $villes = json_decode(file_get_contents($dir.'../static/villes_2012.json'));

    $sites = array();
    $row = 0;
    if (($handle = fopen($dir."../twitter.csv", "r")) !== FALSE) {
      fgetcsv($handle);
      while (($data = fgetcsv($handle)) !== FALSE) {
        $row++;
        if ($row == 1) continue;
        $sites[$data[19]] = explode("|", $data[17]);
        $sites[$data[19]][] = "https://twitter.com/".$data[0];
      }
      fclose($handle);
    }

    if (is_dir($dir)) {
      if ($dh = opendir($dir)) {
        while (($file = readdir($dh)) !== false) {
          if (preg_match('/^\./', $file))
            continue;
    //    echo "$dir$file\n";
          foreach(file($dir.$file) as $line) {
            $json = json_decode($line);
            if (!isset($json->nom) || !strlen($json->nom)) {
              echo "WARNING: ".$dir.$file." doesn't appear to be a json file\n";
              continue;
            }
            $json->nom = trim($json->nom);
            $parl = Doctrine::getTable('Parlementaire')->findOneByNom($json->nom);
            if (!$parl) {
              echo "[INFO] Nouveau député : ".$json->nom." ".$json->url_institution."\n";
              $parl = new Parlementaire();
              $parl->type = 'depute';
              $parl->nom = $json->nom;
              $parl->nom_de_famille = $json->nom_de_famille;
            }
            $parl->sexe = $json->sexe;
            if ($json->date_naissance)
              $parl->date_naissance = $json->date_naissance;
            if ($json->lieu_naissance)
              $parl->lieu_naissance = $json->lieu_naissance;
            if ($json->circonscription)
              $parl->circonscription = $json->circonscription;
            else {
              if ($json->departement)
                $parl->setDepartementParNumero($json->departement);
              if ($json->num_circonscription)
                $parl->num_circo = $json->num_circonscription;
            }
            if ($json->adresses)
              $parl->adresses = $json->adresses;
            if ($json->premiers_mandats)
              $parl->anciens_mandats = $json->premiers_mandats;
            if ($json->autresmandats)
              $parl->autres_mandats = $json->autresmandats;
            if ($json->anciensmandats)
              $parl->anciens_autres_mandats = $json->anciensmandats;
            $parl->debut_mandat = $json->debut_mandat;
            $parl->fin_mandat = $json->fin_mandat;
            if ($json->groupe && (!$parl->fin_mandat || $parl->fin_mandat < $parl->debut_mandat))
              $parl->groupe = $this->splitArrayJson($json->groupe);
            if ($json->parti)
              $parl->parti = $json->parti;
            if ($json->fonctions)
              $parl->fonctions = $this->splitArrayJson($json->fonctions);
            $parl->extras = ($json->extras ? $this->splitArrayJson($json->extras) : array());
            if ($json->groupes)
              $parl->groupes = $this->splitArrayJson($json->groupes);
            if ($json->id_institution)
              $parl->id_an = $json->id_institution;
            if ($json->mails)
              $parl->mails = $json->mails;
            if ($json->photo)
              $parl->photo = $json->photo;
            if ($json->place_hemicycle)
              $parl->place_hemicycle = $json->place_hemicycle;
            if ($json->profession)
              $parl->profession = $json->profession;
            $done_sites = array();
            if ($json->sites_web)
              foreach (array_keys($json->sites_web) as $i) {
                $json->sites_web[$i] = preg_replace("|(://[^/]+)/$|", "$1", trim($json->sites_web[$i]));
                $rootsite = strtolower(preg_replace("#^(https?://|www\.|m\.|fr\.|fr-fr\.)*(.*?)[\s/]*$#", "$2", $json->sites_web[$i]));
                if (!isset($done_sites[$rootsite]))
                  $done_sites[$rootsite] = 1;
                else unset($json->sites_web[$i]);
              }
            if (isset($sites[$parl->slug]) && $sites[$parl->slug]) {
              foreach (array_keys($json->sites_web) as $i)
                if (preg_match("/twitter.com\//", $json->sites_web[$i]) {
                  $rootsite = strtolower(preg_replace("#^(https?://|www\.|m\.|fr\.|fr-fr\.)*(.*?)[\s/]*$#", "$2", $json->sites_web[$i]));
                  unset($done_sites[$rootsite]);
                  unset($json->sites_web[$i]);
                }
              foreach ($sites[$parl->slug] as $site) {
                $site = preg_replace("|(://[^/]+)/$|", "$1", trim($site));
                $rootsite = strtolower(preg_replace("#^(https?://|www\.|m\.|fr\.|fr-fr\.)*(.*?)[\s/]*$#", "$2", $site));
                if (!isset($done_sites[$rootsite]))
                  $json->sites_web[] = $site;
                $done_sites[$rootsite] = 1;
              }
            }
            if ($json->sites_web)
              $parl->sites_web = $json->sites_web;
            else if ($parl->sites_web && !preg_match('/^a:/', $parl->sites_web))
              $parl->sites_web = array($parl->sites_web);
            $parl->collaborateurs = $json->collabs;
            if ($json->url_institution)
              $parl->url_an = $json->url_institution;
            if ($json->suppleant_de)
              $parl->setSuppleantDe($json->suppleant_de);
            if ($json->url_ancien_cpc)
              $parl->url_ancien_cpc = $json->url_ancien_cpc;
            if ($json->url_nouveau_cpc)
              $parl->url_nouveau_cpc = $json->url_nouveau_cpc;
            $parl->villes = $villes->{$parl->getNumDepartement()}->{$parl->num_circo};
            $parl->save();
            if ($json->suppleant) {
              $suppl = Doctrine::getTable('Parlementaire')->findOneByNom(preg_replace('/^\s*M[.mle]* /', '', $json->suppleant));
              if ($suppl) {
                $suppl->setSuppleantDe($parl->nom);
                $suppl->save();
              }
            }
          }
        }
      }
    }
  }
}

{'fraction_non_alphanumeric': 0.18894177956792385, 'fraction_numerical': 0.0037348956426217503, 'mean_word_length': 1.9986824769433464, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 271, 'https://': 2, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '804435', 'n_tokens_mistral': 4539, 'n_tokens_neox': 4357, 'n_words': 675}