<commit_before><?php

class CsvFile
{

  protected $current_line = 0;
  protected $csvdata = null;

  private $file = null;
  private $separator = null;
  private $ignore = null;

  public function getFileName() {
    return $this->file;
  }

  public function __construct($file = null, $options = array()) {
    $this->options = $options;
    if (!isset($this->options["ignore_first_if_comment"])) {
      $this->options["ignore_first_if_comment"] = true;
    }
    $this->ignore = $this->options["ignore_first_if_comment"];
    $this->separator = ';';
    if (!$file)
      return ;
    if (!file_exists($file) && !preg_match('/^http/', $file))
      throw new sfException("Cannont access $file");

    $charset = $this->getCharset($file);
    if ($charset != 'utf-8'){
        exec('iconv -f '.$charset.' -t utf-8//TRANSLIT '.$file.' > '.$file.'.tmp');
        if (filesize($file.".tmp")) {
            exec('mv '.$file.".tmp ".$file);
        }<commit_msg>Tenter une convertion automatique de caractère est trop dangeureux (provoque de pb avec couchdb)<commit_after><?php

class CsvFile
{

  protected $current_line = 0;
  protected $csvdata = null;

  private $file = null;
  private $separator = null;
  private $ignore = null;

  public function getFileName() {
    return $this->file;
  }

  public function __construct($file = null, $options = array()) {
    $this->options = $options;
    if (!isset($this->options["ignore_first_if_comment"])) {
      $this->options["ignore_first_if_comment"] = true;
    }
    $this->ignore = $this->options["ignore_first_if_comment"];
    $this->separator = ';';
    if (!$file)
      return ;
    if (!file_exists($file) && !preg_match('/^http/', $file))
      throw new sfException("Cannont access $file");

    $charset = $this->getCharset($file);
    if ($charset != 'utf-8'){
        exec('iconv -f '.$charset.' -t utf-8 '.$file.' > '.$file.'.tmp');
        if (filesize($file.".tmp")) {
            exec('mv '.$file.".tmp ".$file);
        }
{'fraction_non_alphanumeric': 0.18665328650275967, 'fraction_numerical': 0.0030105368790767687, 'mean_word_length': 3.1198347107438016, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3869848', 'n_tokens_mistral': 674, 'n_tokens_neox': 656, 'n_words': 147}