<commit_before><?php namespace text;
 
/**
 * A stream tokenizer is a tokenizer that works on streams.
 * 
 * Example:
 * ```php
 * $st= new StreamTokenizer(new FileInputStream(new File('test.txt')), " \n");
 * while ($st->hasMoreTokens()) {
 *   printf("- %s\n", $st->nextToken());
 * }
 * ```
 *
 * @test     xp://net.xp_framework.unittest.text.StreamTokenizerTest
 * @see      xp://text.Tokenizer
 */
class StreamTokenizer extends Tokenizer {
  protected
    $_stack = [],
    $_buf   = '',
    $_src   = null;

  /**
   * Reset this tokenizer
   *
   */
  public function reset() {
    $this->_stack= [];
    
    if ($this->_src) {
      if ($this->_src instanceof \io\streams\Seekable) {
        $this->_src->seek(0, SEEK_SET);
      } else {
        throw new \lang\IllegalStateException(
          'Cannot reset, Source '.\xp::stringOf($this->_src).' is not seekable'
        );
      }
    }
    $this->_src= $this->source;
    $this->_buf= '';
  }
  
  /**
   * Tests if there are more tokens available
   *
   * @return  bool more tokens
   */
  public function hasMoreTokens() {
    return !(empty($this->_stack) && false === $this->_buf);
  }
  
  /**
   * Push back a string
   *
   * @param   string str
   */
  public function pushBack($str) {
    $this->_buf= $str.implode('', $this->_stack).$this->_buf;
    $this->_stack= [];
  }
  
  /**
   * Returns the next token from this tokenizer's string
   *
   * @param   bool delimiters default NULL
   * @return  string next token
   */
  public function nextToken($delimiters= null) {
    if (empty($this->_stack)) {
    
      // Read until we have either find a delimiter or until we have 
      // consumed the entire content.
      do {
        $offset= strcspn($this->_buf, $delimiters ? $delimiters : $this->delimiters);
        if ($offset < strlen($this->_buf)- 1 || !$this->_src->available()) break;
        $this->_buf.= $this->_src->read();
      } while (true);

      if (!$this->returnDelims || $offset > 0) $this->_stack[]= substr($this->_buf, 0, $offset);
      if ($this->returnDelims && $offset < strlen($this->_buf)) {
        $this->_stack[]= $this->_buf{$offset};
      }
      $this->_buf= substr($this->_buf, $offset+ 1);
    }
    
    return array_shift($this->_stack);
  }
}<commit_msg>Adjust to behaviour change in substr()<commit_after><?php namespace text;
 
/**
 * A stream tokenizer is a tokenizer that works on streams.
 * 
 * Example:
 * ```php
 * $st= new StreamTokenizer(new FileInputStream(new File('test.txt')), " \n");
 * while ($st->hasMoreTokens()) {
 *   printf("- %s\n", $st->nextToken());
 * }
 * ```
 *
 * @test     xp://net.xp_framework.unittest.text.StreamTokenizerTest
 * @see      xp://text.Tokenizer
 */
class StreamTokenizer extends Tokenizer {
  protected
    $_stack = [],
    $_buf   = '',
    $_src   = null;

  /**
   * Reset this tokenizer
   *
   */
  public function reset() {
    $this->_stack= [];
    
    if ($this->_src) {
      if ($this->_src instanceof \io\streams\Seekable) {
        $this->_src->seek(0, SEEK_SET);
      } else {
        throw new \lang\IllegalStateException(
          'Cannot reset, Source '.\xp::stringOf($this->_src).' is not seekable'
        );
      }
    }
    $this->_src= $this->source;
    $this->_buf= '';
  }
  
  /**
   * Tests if there are more tokens available
   *
   * @return  bool more tokens
   */
  public function hasMoreTokens() {
    return !(empty($this->_stack) && false === $this->_buf);
  }
  
  /**
   * Push back a string
   *
   * @param   string str
   */
  public function pushBack($str) {
    $this->_buf= $str.implode('', $this->_stack).$this->_buf;
    $this->_stack= [];
  }
  
  /**
   * Returns the next token from this tokenizer's string
   *
   * @param   bool delimiters default NULL
   * @return  string next token
   */
  public function nextToken($delimiters= null) {
    if (empty($this->_stack)) {
    
      // Read until we have either find a delimiter or until we have 
      // consumed the entire content.
      do {
        $offset= strcspn($this->_buf, $delimiters ? $delimiters : $this->delimiters);
        if ($offset < strlen($this->_buf)- 1 || !$this->_src->available()) break;
        $this->_buf.= $this->_src->read();
      } while (true);

      if (!$this->returnDelims || $offset > 0) $this->_stack[]= substr($this->_buf, 0, $offset);
      $l= strlen($this->_buf);
      if ($this->returnDelims && $offset < $l) {
        $this->_stack[]= $this->_buf{$offset};
      }
      $offset++;
      $this->_buf= $offset < $l ? substr($this->_buf, $offset) : false;
    }
    
    return array_shift($this->_stack);
  }
}
{'fraction_non_alphanumeric': 0.1796976241900648, 'fraction_numerical': 0.0019438444924406047, 'mean_word_length': 2.4508196721311477, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5618621', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1507, 'n_words': 362}