<commit_before><?php
/* This class is part of the XP framework
 *
 * $Id$
 */

  uses('text.doclet.RootDoc');

  define('ST_LOOKING_FOR_CLASS',          'looking:class');
  define('ST_FUNCTION_ARGS',              'function-args');
  define('ST_USES_LIST',                  'uses-list');
  
  /**
   * Rewrites sourececode
   *
   * @purpose  Utility
   */
  class SourceRewriter extends Object {
    public 
      $names= NULL;
    
    /**
     * Set name mapping. Required operation
     *
     * @param   net.xp_framework.tools.vm.util.NameMapping mapping
     */
    public function setNameMapping($mapping) {
      $this->names= $mapping;
    }
    
    /**
     * Rewrite tokens
     *
     * @param   array tokens as returned by token_get_all()
     * @param   bool debug default FALSE
     * @return  string
     * @throws  lang.Exception to indicate rewriting failures
     */
    public function rewrite($tokens, $debug= FALSE) {
      static $map= array(
        'uses'        => T_USES, 
        'define'      => T_DEFINE,
      );

      $out= '';
      $states= array(ST_INITIAL);
      $skip= FALSE;
      $brackets= 0;

      // Compile list of classes to be added to uses()
      $used= array();
      $this->names->current->usedClasses->rewind();
      while ($this->names->current->usedClasses->hasNext()) {
        $class= $this->names->current->usedClasses->next();
        $used[]= strtr($this->names->packagedNameOf($class->qualifiedName()), NS_SEPARATOR, '.');
      }
      $this->names->current->interfaces->rewind();
      while ($this->names->current->interfaces->hasNext()) {
        $interface= $this->names->current->interfaces->next();
        $used[]= strtr($this->names->packagedNameOf($interface->qualifiedName()), NS_SEPARATOR, '.');
      }

      for ($i= 0, $s= sizeof($tokens); $i < $s; $i++) {
        $t= $tokens[$i];
        if (is_array($t) && isset($map[$t[1]])) $t[0]= $map[$t[1]];

        // Debug information
        if ($debug) {
          if (is_array($tokens[$i])) {
            Console::writeLinef('%-20s: %-30s %s', $states[0], token_name($tokens[$i][0]), $tokens[$i][1]);
          } else {
            Console::writeLinef('%-20s: %-30s %s', $states[0], 'T_NONE', $tokens[$i]);
          }
        }

        // State-based actions
        switch ($states[0].$t[0]) {
          case ST_INITIAL.T_OPEN_TAG:
            $t= '';
            break;

          case ST_INITIAL.T_CLOSE_TAG:
            $t= '';
            break;

          case ST_INITIAL.T_USES:
            $skip= TRUE;
            array_unshift($states, ST_USES_LIST);
            break;

          case ST_USES_LIST.';':
            $skip= FALSE;
            $t= '';
            array_shift($states);
            break;

          case ST_INITIAL.T_DOC_COMMENT:
            $qualified= $this->names->current->qualifiedName();
            $package= substr($qualified, 0, strrpos($qualified, '.'));

            sort($used);
            $uses= '';
            foreach ($used as $classname) {
              $uses.= 'import '.$classname.";\n";
            }
            $out= rtrim($out)."\n\n".$uses."\npackage ".$this->names->packagedNameOf($package)." {\n\n  ";
            break;

          case ST_INITIAL.T_CLASS:
            if (!$package) {                              // Class apidoc was missing!
              $qualified= $this->names->current->qualifiedName();
              $package= substr($qualified, 0, strrpos($qualified, '.'));

              sort($used);
              $uses= '';
              foreach ($used as $classname) {
                $uses.= 'import '.$classname.";\n";
              }
              $out= rtrim($out)."\n\n".$uses."\n\npackage ".$this->names->packagedNameOf($package)." {\n\n  ";
            }
            array_unshift($states, ST_CLASS);
            break;

          case ST_CLASS.'{':
            $skip= FALSE;
            array_unshift($states, ST_CLASS_BODY);
            $brackets[ST_CLASS_BODY]= 1;
            break;

          case ST_CLASS_BODY.'}':
            array_shift($states);
            $t.= "\n}"; // One extra for package statement
            array_shift($states);<commit_msg>- Add uses inside package<commit_after><?php
/* This class is part of the XP framework
 *
 * $Id$
 */

  uses('text.doclet.RootDoc');

  define('ST_LOOKING_FOR_CLASS',          'looking:class');
  define('ST_FUNCTION_ARGS',              'function-args');
  define('ST_USES_LIST',                  'uses-list');
  
  /**
   * Rewrites sourececode
   *
   * @purpose  Utility
   */
  class SourceRewriter extends Object {
    public 
      $names= NULL;
    
    /**
     * Set name mapping. Required operation
     *
     * @param   net.xp_framework.tools.vm.util.NameMapping mapping
     */
    public function setNameMapping($mapping) {
      $this->names= $mapping;
    }
    
    /**
     * Rewrite tokens
     *
     * @param   array tokens as returned by token_get_all()
     * @param   bool debug default FALSE
     * @return  string
     * @throws  lang.Exception to indicate rewriting failures
     */
    public function rewrite($tokens, $debug= FALSE) {
      static $map= array(
        'uses'        => T_USES, 
        'define'      => T_DEFINE,
      );

      $out= '';
      $states= array(ST_INITIAL);
      $skip= FALSE;
      $brackets= 0;

      // Compile list of classes to be added to uses()
      $used= array();
      $this->names->current->usedClasses->rewind();
      while ($this->names->current->usedClasses->hasNext()) {
        $class= $this->names->current->usedClasses->next();
        $used[]= strtr($this->names->packagedNameOf($class->qualifiedName()), NS_SEPARATOR, '.');
      }
      $this->names->current->interfaces->rewind();
      while ($this->names->current->interfaces->hasNext()) {
        $interface= $this->names->current->interfaces->next();
        $used[]= strtr($this->names->packagedNameOf($interface->qualifiedName()), NS_SEPARATOR, '.');
      }

      for ($i= 0, $s= sizeof($tokens); $i < $s; $i++) {
        $t= $tokens[$i];
        if (is_array($t) && isset($map[$t[1]])) $t[0]= $map[$t[1]];

        // Debug information
        if ($debug) {
          if (is_array($tokens[$i])) {
            Console::writeLinef('%-20s: %-30s %s', $states[0], token_name($tokens[$i][0]), $tokens[$i][1]);
          } else {
            Console::writeLinef('%-20s: %-30s %s', $states[0], 'T_NONE', $tokens[$i]);
          }
        }

        // State-based actions
        switch ($states[0].$t[0]) {
          case ST_INITIAL.T_OPEN_TAG:
            $t= '';
            break;

          case ST_INITIAL.T_CLOSE_TAG:
            $t= '';
            break;

          case ST_INITIAL.T_USES:
            $skip= TRUE;
            array_unshift($states, ST_USES_LIST);
            break;

          case ST_USES_LIST.';':
            $skip= FALSE;
            $t= '';
            array_shift($states);
            break;

          case ST_INITIAL.T_DOC_COMMENT:
            $qualified= $this->names->current->qualifiedName();
            $package= substr($qualified, 0, strrpos($qualified, '.'));

            sort($used);
            $uses= '';
            foreach ($used as $classname) {
              $uses.= '  import '.$classname.";\n";
            }
            $out= rtrim($out)."\npackage ".$this->names->packagedNameOf($package)." {\n".$uses."\n  ";
            break;

          case ST_INITIAL.T_CLASS:
          case ST_INITIAL.T_INTERFACE:
            if (!$package) {                              // Class apidoc was missing!
              $qualified= $this->names->current->qualifiedName();
              $package= substr($qualified, 0, strrpos($qualified, '.'));

              sort($used);
              $uses= '';
              foreach ($used as $classname) {
                $uses.= '  import '.$classname.";\n";
              }
              $out= rtrim($out)."\npackage ".$this->names->packagedNameOf($package)." {\n".$uses."\n  ";
            }
            array_unshift($states, ST_CLASS);
            break;

          case ST_CLASS.'{':
            $skip= FALSE;
            array_unshift($states, ST_CLASS_BODY);
            $brackets[ST_CLASS_BODY]= 1;
            break;

          case ST_CLASS_BODY.'}':
            array_shift($states);
            $t.= "\n}"; // One extra for package statement
            array_shift($states);
{'fraction_non_alphanumeric': 0.16832038254632398, 'fraction_numerical': 0.005260011954572624, 'mean_word_length': 1.8073825503355705, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5715115', 'n_tokens_mistral': 2650, 'n_tokens_neox': 2538, 'n_words': 487}