<commit_before><?php
/* This class is part of the XP framework
 *
 * $Id$ 
 */

  uses(
    'unittest.TestCase',
    'text.Tokenizer'
  );

  /**
   * Test string tokenizing.
   *
   * @see      xp://text.Tokenizer
   * @purpose  Abstract base class for different tokenizer tests
   */
  abstract class TokenizerTest extends TestCase {
  
    /**
     * Retrieve a tokenizer instance
     *
     * @param   string source
     * @param   string delimiters default ' '
     * @param   bool returnDelims default FALSE
     * @return  text.Tokenizer
     */
    protected abstract function tokenizerInstance($source, $delimiters= ' ', $returnDelims= FALSE);
  
    /**
     * Test string tokenizing
     *
     */
    #[@test]
    public function testSimpleString() {
      $t= $this->tokenizerInstance("Hello World!\nThis is an example", " \n");
      $this->assertEquals('Hello', $t->nextToken());
      $this->assertEquals('World!', $t->nextToken());
      $this->assertEquals('This', $t->nextToken());
      $this->assertEquals('is', $t->nextToken());
      $this->assertEquals('an', $t->nextToken());
      $this->assertEquals('example', $t->nextToken());
      $this->assertFalse($t->hasMoreTokens());
    }

    /**
     * Test string tokenizing
     *
     */
    #[@test]
    public function testSimpleStringWithDelims() {
      $t= $this->tokenizerInstance("Hello World!\nThis is an example", " \n", TRUE);
      $this->assertEquals('Hello', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals('World!', $t->nextToken());
      $this->assertEquals("\n", $t->nextToken());
      $this->assertEquals('This', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals('is', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals('an', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals('example', $t->nextToken());
      $this->assertFalse($t->hasMoreTokens());
    }
    
    /**
     * Test string tokenizing
     *
     */
    #[@test]
    public function repetetiveDelimiters() {
      $t= $this->tokenizerInstance("Hello \nWorld!", " \n");
      $this->assertEquals('Hello', $t->nextToken());
      $this->assertEquals('', $t->nextToken());
      $this->assertEquals('World!', $t->nextToken());
      $this->assertFalse($t->hasMoreTokens());
    }

    /**
     * Test string tokenizing
     *
     */
    #[@test]
    public function repetetiveDelimitersWithDelims() {
      $t= $this->tokenizerInstance("Hello \nWorld!", " \n", TRUE);
      $this->assertEquals('Hello', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals("\n", $t->nextToken());
      $this->assertEquals('World!', $t->nextToken());
      $this->assertFalse($t->hasMoreTokens());
    }
  }
?><commit_msg>- Test different iteration mechanisms - Test reset() method<commit_after><?php
/* This class is part of the XP framework
 *
 * $Id$ 
 */

  uses(
    'unittest.TestCase',
    'text.Tokenizer'
  );

  /**
   * Test string tokenizing.
   *
   * @see      xp://text.Tokenizer
   * @purpose  Abstract base class for different tokenizer tests
   */
  abstract class TokenizerTest extends TestCase {
  
    /**
     * Retrieve a tokenizer instance
     *
     * @param   string source
     * @param   string delimiters default ' '
     * @param   bool returnDelims default FALSE
     * @return  text.Tokenizer
     */
    protected abstract function tokenizerInstance($source, $delimiters= ' ', $returnDelims= FALSE);
  
    /**
     * Test string tokenizing
     *
     */
    #[@test]
    public function testSimpleString() {
      $t= $this->tokenizerInstance("Hello World!\nThis is an example", " \n");
      $this->assertEquals('Hello', $t->nextToken());
      $this->assertEquals('World!', $t->nextToken());
      $this->assertEquals('This', $t->nextToken());
      $this->assertEquals('is', $t->nextToken());
      $this->assertEquals('an', $t->nextToken());
      $this->assertEquals('example', $t->nextToken());
      $this->assertFalse($t->hasMoreTokens());
    }

    /**
     * Test string tokenizing
     *
     */
    #[@test]
    public function testSimpleStringWithDelims() {
      $t= $this->tokenizerInstance("Hello World!\nThis is an example", " \n", TRUE);
      $this->assertEquals('Hello', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals('World!', $t->nextToken());
      $this->assertEquals("\n", $t->nextToken());
      $this->assertEquals('This', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals('is', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals('an', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals('example', $t->nextToken());
      $this->assertFalse($t->hasMoreTokens());
    }
    
    /**
     * Test string tokenizing
     *
     */
    #[@test]
    public function repetetiveDelimiters() {
      $t= $this->tokenizerInstance("Hello \nWorld!", " \n");
      $this->assertEquals('Hello', $t->nextToken());
      $this->assertEquals('', $t->nextToken());
      $this->assertEquals('World!', $t->nextToken());
      $this->assertFalse($t->hasMoreTokens());
    }

    /**
     * Test string tokenizing
     *
     */
    #[@test]
    public function repetetiveDelimitersWithDelims() {
      $t= $this->tokenizerInstance("Hello \nWorld!", " \n", TRUE);
      $this->assertEquals('Hello', $t->nextToken());
      $this->assertEquals(' ', $t->nextToken());
      $this->assertEquals("\n", $t->nextToken());
      $this->assertEquals('World!', $t->nextToken());
      $this->assertFalse($t->hasMoreTokens());
    }
    
    /**
     * Test for loop iteration
     *
     */
    #[@test]
    public function forIteration() {
      $r= array();
      for ($t= $this->tokenizerInstance('A B C', ' '); $t->hasMoreTokens(); ) {
        $r[]= $t->nextToken();
      }
      $this->assertEquals(range('A', 'C'), $r);
    }

    /**
     * Test while loop iteration
     *
     */
    #[@test]
    public function whileIteration() {
      $r= array();
      $t= $this->tokenizerInstance('A B C', ' ');
      while ($t->hasMoreTokens()) {
        $r[]= $t->nextToken();
      }
      $this->assertEquals(range('A', 'C'), $r);
    }

    /**
     * Test foreach() overloading
     *
     */
    #[@test]
    public function foreachIteration() {
      $r= array();
      foreach ($this->tokenizerInstance('A B C', ' ') as $token) {
        $r[]= $token;
      }
      $this->assertEquals(range('A', 'C'), $r);
    }

    /**
     * Test resetting a tokenizer
     *
     */
    #[@test]
    public function reset() {
      $t= $this->tokenizerInstance('A B C', ' ');
      $this->assertTrue($t->hasMoreTokens());
      $this->assertEquals('A', $t->nextToken());
      $t->reset();
      $this->assertTrue($t->hasMoreTokens());
      $this->assertEquals('A', $t->nextToken());
    }
  }
?>
{'fraction_non_alphanumeric': 0.205640589569161, 'fraction_numerical': 0.0, 'mean_word_length': 2.72401055408971, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 145, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7660921', 'n_tokens_mistral': 2168, 'n_tokens_neox': 2165, 'n_words': 408}