<commit_before><?php
/* This class is part of the XP framework
 *
 * $Id$
 */
 
  define('T_NONE',			0x0000);    // None of the known tokens, CDATA only
  define('T_ANY',			0xFFFF);    // Any token
 
  /**
   * PHP Tokenizer
   * Class wrapper around PHPs tokenizer extension
   * 
   * @ext   tokenize
   * @see   php://tokenizer
   * @see   lang.apidoc.parser.GenericParser
   */ 
  class PHPTokenizer extends Object {
    var 
      $tokens= array(),
      $rules=  array();
      
    var
      $_offset,
      $_size;
      
    /**
     * Set tokens
     *
     * @access  public
     * @param   string str a string containing all of the tokens
     */
    function setTokenString($str) {
      $this->tokens= token_get_all($str);
      $this->_offset= 0;
      $this->_size= sizeof($this->tokens);
    }
    
    /**
     * Set tokens
     *
     * @access  public
     * @param   array tokens
     */
    function setTokens($tokens) {
      $this->tokens= $tokens;
      $this->_offset= 0;
      $this->_size= sizeof($tokens);
    }
    
    /**
     * Gets a token's name
     *
     * @access  public
     * @param   int tok Token constant, e.g. T_WHITESPACE
     * @return  string name
     */
    function getTokenName($tok) {
      switch ($tok) {
        case T_NONE: return 'T_NONE';
        case T_ANY: return 'T_ANY';
      }
      return token_name($tok);
    }
    
    /**
     * Adds a rule which can be applied with applyRules()
     *
     * When a list of matches has succeeded in applying, the defined callback
     * function is called with according parameters. Parameters may be any of
     * PHPs known datatypes in their notation or the special $X syntax:
     * 
     * Consider the following parameters:
     * <code>
     *   $name=     'function_with_comment';
     *   $match=    array('T_COMMENT', 'T_WHITESPACE', 'T_FUNCTION', 'T_WHITESPACE', 'T_STRING');
     *   $callback= 'setFunctionComment';
     *   $params=   array('$5', '$1', FALSE);
     * </code>
     *
     * In this example, setFunctionComment() is called with the functions name (the 
     * 5th token, counting from 1) as its first parameter, the comment as its second
     * and boolean FALSE as its third parameter
     *
     * @see     util.text.PHPTokenizer#applyRules
     * @access  public
     * @param   string name rule name
     * @param   array match list of tokens to match
     * @param   mixed callback either a string or array(&$obj, 'function') syntax
     * @param   array params parameters for callback
     */
    function addRule($name, $match, $callback, $params) {
      $this->rules[$name]= array(
        'expect'        => 0,
        'match'         => $match, 
        'callback'      => $callback, 
        'params'        => $params
      );
    }

    /**
     * Private function which always returns an array for a token
     *
     * @access  private
     * @param   int i offset
     * @return  array tokendata (type, cdata)
     */
    function _token($i) {
      return (is_array($this->tokens[$i])
        ? $this->tokens[$i]
        : array(T_NONE, $this->tokens[$i])
      );
    }
    
    /**
     * Get first token
     *
     * @access  public
     * @return  array first token
     */
    function getFirstToken() {
      return $this->_token($this->_offset= 0);
    }
    
    /**
     * Get next token
     *
     * @access  public
     * @return  array next token from current offset or FALSE when no more tokens exist
     */
    function getNextToken() {
      if (++$this->_offset >= $this->_size) return FALSE;
      return $this->_token($this->_offset);
    }
    
    /**
     * Apply rules on all tokens
     *
     * @access  public
     * @param   util.log.LogCategory CAT default NULL a log category to print debug to
     * @return  bool success
     */
    function applyRules($CAT= NULL) {
      $data= array();
      
      // Loop throught tokens
      $tok= $this->getFirstToken();
      $i= 0;
      do {
        list($token, $cdata)= $tok;

        // Go through all rules and see if one matches
        reset($this->rules);
        $name= key($this->rules);
	    do {
          $CAT && $CAT->debugf('[%04x:%-18s] >>> %s "%s"', $i, $name, $this->getTokenName($token), $cdata);<commit_msg>- Make _token public and rename it to getToken()<commit_after><?php
/* This class is part of the XP framework
 *
 * $Id$
 */
 
  define('T_NONE',			0x0000);    // None of the known tokens, CDATA only
  define('T_ANY',			0xFFFF);    // Any token
 
  /**
   * PHP Tokenizer
   * Class wrapper around PHPs tokenizer extension
   * 
   * @ext   tokenize
   * @see   php://tokenizer
   * @see   lang.apidoc.parser.GenericParser
   */ 
  class PHPTokenizer extends Object {
    var 
      $tokens= array(),
      $rules=  array();
      
    var
      $_offset,
      $_size;
      
    /**
     * Set tokens
     *
     * @access  public
     * @param   string str a string containing all of the tokens
     */
    function setTokenString($str) {
      $this->tokens= token_get_all($str);
      $this->_offset= 0;
      $this->_size= sizeof($this->tokens);
    }
    
    /**
     * Set tokens
     *
     * @access  public
     * @param   array tokens
     */
    function setTokens($tokens) {
      $this->tokens= $tokens;
      $this->_offset= 0;
      $this->_size= sizeof($tokens);
    }
    
    /**
     * Gets a token's name
     *
     * @access  public
     * @param   int tok Token constant, e.g. T_WHITESPACE
     * @return  string name
     */
    function getTokenName($tok) {
      switch ($tok) {
        case T_NONE: return 'T_NONE';
        case T_ANY: return 'T_ANY';
      }
      return token_name($tok);
    }
    
    /**
     * Adds a rule which can be applied with applyRules()
     *
     * When a list of matches has succeeded in applying, the defined callback
     * function is called with according parameters. Parameters may be any of
     * PHPs known datatypes in their notation or the special $X syntax:
     * 
     * Consider the following parameters:
     * <code>
     *   $name=     'function_with_comment';
     *   $match=    array('T_COMMENT', 'T_WHITESPACE', 'T_FUNCTION', 'T_WHITESPACE', 'T_STRING');
     *   $callback= 'setFunctionComment';
     *   $params=   array('$5', '$1', FALSE);
     * </code>
     *
     * In this example, setFunctionComment() is called with the functions name (the 
     * 5th token, counting from 1) as its first parameter, the comment as its second
     * and boolean FALSE as its third parameter
     *
     * @see     util.text.PHPTokenizer#applyRules
     * @access  public
     * @param   string name rule name
     * @param   array match list of tokens to match
     * @param   mixed callback either a string or array(&$obj, 'function') syntax
     * @param   array params parameters for callback
     */
    function addRule($name, $match, $callback, $params) {
      $this->rules[$name]= array(
        'expect'        => 0,
        'match'         => $match, 
        'callback'      => $callback, 
        'params'        => $params
      );
    }

    /**
     * Return a token by position
     *
     * @access  public
     * @param   int i offset
     * @return  array tokendata (type, cdata)
     */
    function getToken($i) {
      return (is_array($this->tokens[$i])
        ? $this->tokens[$i]
        : array(T_NONE, $this->tokens[$i])
      );
    }
    
    /**
     * Get first token
     *
     * @access  public
     * @return  array first token
     */
    function getFirstToken() {
      return $this->getToken($this->_offset= 0);
    }
    
    /**
     * Get next token
     *
     * @access  public
     * @return  array next token from current offset or FALSE when no more tokens exist
     */
    function getNextToken() {
      if (++$this->_offset >= $this->_size) return FALSE;
      return $this->getToken($this->_offset);
    }
    
    /**
     * Apply rules on all tokens
     *
     * @access  public
     * @param   util.log.LogCategory CAT default NULL a log category to print debug to
     * @return  bool success
     */
    function applyRules($CAT= NULL) {
      $data= array();
      
      // Loop throught tokens
      $tok= $this->getFirstToken();
      $i= 0;
      do {
        list($token, $cdata)= $tok;

        // Go through all rules and see if one matches
        reset($this->rules);
        $name= key($this->rules);
	    do {
          $CAT && $CAT->debugf('[%04x:%-18s] >>> %s "%s"', $i, $name, $this->getTokenName($token), $cdata);
{'fraction_non_alphanumeric': 0.12687793427230046, 'fraction_numerical': 0.004460093896713615, 'mean_word_length': 1.9597082320250088, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1868002', 'n_tokens_mistral': 2668, 'n_tokens_neox': 2595, 'n_words': 778}