<commit_before><?php
/* This class is part of the XP framework
 *
 * $Id$
 */
 
  /**
   * A string tokenizer allows you to break a string into tokens,
   * these being delimited by any character in the delimiter.
   * 
   * Example:
   * <code>
   *   $st= &new StringTokenizer("Hello World!\nThis is an example", " \n");
   *   while ($st->hasMoreTokens()) {
   *     printf("- %s\n", $st->nextToken());
   *   }
   * </code>
   *
   * This would output:
   * <pre>
   *   - Hello
   *   - World!
   *   - This
   *   - is
   *   - an
   *   - example
   * </pre>
   *
   * @see php://strtok
   */
  class StringTokenizer extends Object {
    var $delim, $tok;
    
    /**
     * Constructor
     *
     * @access  public
     * @param   string str
     * @param   string delim default ' '
     */
    function __construct($str, $delim= ' ') {
      $this->delim= $delim;
      $this->tok= strtok($str, $this->delim);
      parent::__construct();
    }
    
    /**
     * Tests if there are more tokens available
     *
     * @access  public
     * @return  bool more tokens
     */
    function hasMoreTokens() {
      return ($this->tok !== FALSE);
    }
    
    /**
     * Returns the next token from this tokenizer's string
     *
     * @access  public
     * @return  string next token
     */
    function nextToken() {
      $tok= $this->tok;
      $this->tok= strtok($this->delim);
      return $tok;
    }
  }
?><commit_msg>- Add feature to include delimiters in tokens returned from nextToken()<commit_after><?php
/* This class is part of the XP framework
 *
 * $Id$
 */
 
  /**
   * A string tokenizer allows you to break a string into tokens,
   * these being delimited by any character in the delimiter.
   * 
   * Example:
   * <code>
   *   $st= &new StringTokenizer("Hello World!\nThis is an example", " \n");
   *   while ($st->hasMoreTokens()) {
   *     printf("- %s\n", $st->nextToken());
   *   }
   * </code>
   *
   * This would output:
   * <pre>
   *   - Hello
   *   - World!
   *   - This
   *   - is
   *   - an
   *   - example
   * </pre>
   *
   * @see php://strtok
   */
  class StringTokenizer extends Object {
    var 
      $delimiters   = '',
      $returnDelims = FALSE;
    
    var
      $_str         = '',
      $_stack       = array();
    
    /**
     * Constructor
     *
     * @access  public
     * @param   string str
     * @param   string delimiter default ' '
     * @param   bool returnDelims default FALSE
     */
    function __construct($str, $delimiters= ' ', $returnDelims= FALSE) {
      $this->delimiters= $delimiters;
      $this->returnDelims= $returnDelims;
      $this->_str= $str;
    }
    
    /**
     * Tests if there are more tokens available
     *
     * @access  public
     * @return  bool more tokens
     */
    function hasMoreTokens() {
      return strlen($this->_str) > 0;
    }
    
    /**
     * Returns the next token from this tokenizer's string
     *
     * @access  public
     * @return  string next token
     */
    function nextToken($delimiters= NULL) {
      if (empty($this->_stack)) {
        $offset= strcspn($this->_str, $delimiters ? $delimiters : $this->delimiters);
        $this->_stack[]= substr($this->_str, 0, $offset);
        if ($this->returnDelims && $offset < strlen($this->_str)) {
          $this->_stack[]= $this->_str{$offset};
        }
        $this->_str= substr($this->_str, $offset+ 1);
      }
      return array_shift($this->_stack);
    }
  }
?>
{'fraction_non_alphanumeric': 0.14939550949913644, 'fraction_numerical': 0.0008635578583765112, 'mean_word_length': 1.8790389395194698, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1773902', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1130, 'n_words': 290}