<commit_before><?php

class CsvFile 
{

  private $file = null;
  private $separator = null;
  protected $csvdata = null;
  private $ignore = null;

  public function getFileName() {
    return $this->file;
  }

  public function __construct($file = null, $ignore_first_if_comment = 1) {
    $this->ignore = $ignore_first_if_comment;
    $this->separator = ';';
    if (!$file)
      return ;
    if (!file_exists($file) && !preg_match('/^http/', $file))
      throw new Exception("Cannont access $file");
    $this->file = $file;
    $handle = fopen($this->file, 'r');
    if (!$handle) {
      throw new Exception('invalid_file');
    }
    $buffer = fread($handle, 500);
    fclose($handle);
    $buffer = preg_replace('/$[^\n]*\n/', '', $buffer);
    if (!$buffer) {
      throw new Exception('invalid_file');
    }
    $virgule = explode(',', $buffer);
    $ptvirgule = explode(';', $buffer);
    if (count($virgule) > count($ptvirgule))
      $this->separator = ',';
  }

  public function getCsv() {<commit_msg>Gestion de la tabulation (même si pas très utile)<commit_after><?php

class CsvFile 
{

  private $file = null;
  private $separator = null;
  protected $csvdata = null;
  private $ignore = null;

  public function getFileName() {
    return $this->file;
  }

  public function __construct($file = null, $ignore_first_if_comment = 1) {
    $this->ignore = $ignore_first_if_comment;
    $this->separator = ';';
    if (!$file)
      return ;
    if (!file_exists($file) && !preg_match('/^http/', $file))
      throw new Exception("Cannont access $file");
    $this->file = $file;
    $handle = fopen($this->file, 'r');
    if (!$handle) {
      throw new Exception('invalid_file');
    }
    $buffer = fread($handle, 500);
    fclose($handle);
    $buffer = preg_replace('/$[^\n]*\n/', '', $buffer);
    if (!$buffer) {
      throw new Exception('invalid_file');
    }

    $virgule = explode(',', $buffer);
    $ptvirgule = explode(';', $buffer);
    $tabulation = explode('\t', $buffer);
    if (count($virgule) > count($ptvirgule) && count($virgule) > count($tabulation))
      $this->separator = ',';
    else if (count($tabulation) > count($ptvirgule))
      $this->separator = '\t';
  }

  public function getCsv() {
{'fraction_non_alphanumeric': 0.18579723090665476, 'fraction_numerical': 0.003573023671281822, 'mean_word_length': 3.2829827915869982, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6168603', 'n_tokens_mistral': 773, 'n_tokens_neox': 758, 'n_words': 171}