<commit_before><?php
/* This class is part of the XP framework
 *
 * $Id$ 
 */

  uses(
    'unittest.TestCase',
    'text.parser.CSVParser'
  );

  /**
   * Test string tokenizing.
   *
   * @see      xp://text.parser.CSVParser
   * @purpose  TestCase
   */
  class CSVParserTest extends TestCase {
  
    /**
     * Setup
     *
     */
    public function setup() {
      $this->stream= new Stream();
      $this->parser= new CSVParser();
      $this->parser->setInputStream($this->stream);
    }

    /**
     * Test guess of delimiters
     *
     */
    #[@test]
    function testGuessDelimiter() {
      $this->stream->write('a,b,c,d,e,f');
      $this->stream->rewind();
      
      $this->assertEquals(',', $this->parser->guessDelimiter());
    }

    /**
     * Test record with escape character
     *
     */
    #[@test]
    public function testRecordWithEscapes() {
      $this->stream->write('"UTDI.DE",13.56,-0.63,"9:16am","2/28/2007",2274004');
      $this->stream->rewind();
      $this->parser->setColDelimiter(',');
      
      with ($record= $this->parser->getNextRecord()); {
        $this->assertEquals('UTDI.DE', array_shift($record));
        $this->assertEquals('13.56', array_shift($record));
        $this->assertEquals('-0.63', array_shift($record));
        $this->assertEquals('9:16am', array_shift($record));
        $this->assertEquals('2/28/2007', array_shift($record));
        $this->assertEquals('2274004', array_shift($record));
      }
    }
  }
?><commit_msg>- make secret changeable<commit_after><?php
/* This class is part of the XP framework
 *
 * $Id$ 
 */

  uses(
    'unittest.TestCase',
    'text.parser.CSVParser'
  );

  /**
   * Test string tokenizing.
   *
   * @see      xp://text.parser.CSVParser
   * @purpose  TestCase
   */
  class CSVParserTest extends TestCase {
  
    /**
     * Setup
     *
     */
    public function setup() {
      $this->stream= new Stream();
      $this->parser= new CSVParser();
      $this->parser->setInputStream($this->stream);
    }

    /**
     * Test guess of delimiters
     *
     */
    #[@test]
    function testGuessDelimiter() {
      $this->stream->write('a,b,c,d,e,f');
      $this->stream->rewind();
      
      $this->assertEquals(',', $this->parser->guessDelimiter());
    }

    /**
     * Test record with escape character
     *
     */
    #[@test]
    public function testRecordWithEscapes() {
      $this->stream->write('"UTDI.DE",13.56,-0.63,"9:16am","2/28/2007",2274004');
      $this->stream->rewind();
      $this->parser->setColDelimiter(',');
      
      with ($record= $this->parser->getNextRecord()); {
        $this->assertEquals('UTDI.DE', array_shift($record));
        $this->assertEquals('13.56', array_shift($record));
        $this->assertEquals('-0.63', array_shift($record));
        $this->assertEquals('9:16am', array_shift($record));
        $this->assertEquals('2/28/2007', array_shift($record));
        $this->assertEquals('2274004', array_shift($record));
      }
    }
    
    /**
     * Test
     *
     */
    #[@test]
    public function lastColumn() {
      $this->stream->write('foo|bar|baz|'.PHP_EOL);
      $this->stream->write('bla|bla|bla|'.PHP_EOL);
      $this->stream->write('bla|bla|bla|'.PHP_EOL);
      $this->stream->rewind();
      $this->parser->setColDelimiter('|');
      $this->parser->getHeaderRecord();
      
      $this->assertEquals(array(
        'foo' => 'bla',
        'bar' => 'bla',
        'baz' => 'bla'
        ), $this->parser->getNextRecord()
      );
      $this->assertEquals(array(
        'foo' => 'bla',
        'bar' => 'bla',
        'baz' => 'bla'
        ), $this->parser->getNextRecord()
      );
    }    
  }
?>
{'fraction_non_alphanumeric': 0.19983775013520821, 'fraction_numerical': 0.025959978366684695, 'mean_word_length': 2.362727272727273, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6818650', 'n_tokens_mistral': 1295, 'n_tokens_neox': 1219, 'n_words': 176}