<commit_before><?php
/* This class is part of the XP framework
 *
 * $Id$
 */

  uses('text.doclet.markup.MarkupProcessor');

  /**
   * Processes <code> ... </code>
   *
   * @purpose  Processor
   */
  class CodeProcessor extends MarkupProcessor {
    public
      $buffer = '';

    /**
     * Initializes the processor.
     *
     * @return  string
     */
    public function initialize() {
      $this->buffer= '';
      return '';
    }
    
    /**
     * Process a token
     *
     * @param   string token
     * @return  string
     */
    public function process($token) {
      $this->buffer.= $token;
      return '';
    }

    /**
     * Finalizes the processor.
     *
     * @return  string
     */    
    public function finalize() {
      static $classes= array(
        T_VARIABLE                      => 'variable',

        T_CLASS                         => 'keyword',
        T_INTERFACE                     => 'keyword',
        T_EXTENDS                       => 'keyword',
        T_IMPLEMENTS                    => 'keyword',
        T_CATCH                         => 'keyword',
        T_THROW                         => 'keyword',
        T_TRY                           => 'keyword',
        T_NEW                           => 'keyword',
        T_FUNCTION                      => 'keyword',
        T_FOR                           => 'keyword',
        T_IF                            => 'keyword',
        T_ELSE                          => 'keyword',
        T_SWITCH                        => 'keyword',
        T_WHILE                         => 'keyword',
        T_FOREACH                       => 'keyword',
        T_RETURN                        => 'keyword',
        
        T_STATIC                        => 'modifier',
        T_ABSTRACT                      => 'modifier',
        T_PUBLIC                        => 'modifier',
        T_PRIVATE                       => 'modifier',
        T_PROTECTED                     => 'modifier',
        T_FINAL                         => 'modifier',
        
        T_DNUMBER                       => 'number',
        T_LNUMBER                       => 'number',
        T_CONSTANT_ENCAPSED_STRING      => 'string',
        
        T_COMMENT                       => 'comment',
        
        '{'                             => 'bracket',
        '}'                             => 'bracket',
        '('                             => 'bracket',
        ')'                             => 'bracket',
      );

      $current= 'default';
      $out= '';
      
      // Tokenize buffer
      $tokens= token_get_all('<?php '.trim($this->buffer, "\r\n").'?>');
      if (!is_array($tokens) || xp::errorAt(__FILE__, __LINE__ - 1)) {
        throw new FormatException('Cannot parse "'.$this->buffer.'"');
      }
      
      // Create HTML
      foreach ($tokens as $token) {
        $class= (isset($classes[$token[0]]) 
          ? $classes[$token[0]]
          : 'default'
        );
        
        // Handle annotations
        if (is_array($token) && T_COMMENT === $token[0] && '#' === $token[1][0]) {
          $class= 'annotation';
        }
        
        if ($current != $class) {
          $out.= '</span><span class="'.$class.'">';
          $current= $class;
        }

        $out.= strtr(htmlspecialchars(is_array($token) ? $token[1] : $token), array(
          "\n" => '<br/>',
          "\r" => ''
        ));
      }
      
      return '</p><pre class="code"><span>'.substr($out, 9, -5).'</span></pre><p>';
    }
  }
?><commit_msg>- Ensure <code></code> does not produce illegal XML - Change way how we strip "<?php " and " ?>"<commit_after><?php
/* This class is part of the XP framework
 *
 * $Id$
 */

  uses('text.doclet.markup.MarkupProcessor');

  /**
   * Processes <code> ... </code>
   *
   * @purpose  Processor
   */
  class CodeProcessor extends MarkupProcessor {
    public
      $buffer = '';

    /**
     * Initializes the processor.
     *
     * @return  string
     */
    public function initialize() {
      $this->buffer= '';
      return '';
    }
    
    /**
     * Process a token
     *
     * @param   string token
     * @return  string
     */
    public function process($token) {
      $this->buffer.= $token;
      return '';
    }

    /**
     * Finalizes the processor.
     *
     * @return  string
     */    
    public function finalize() {
      static $classes= array(
        T_VARIABLE                      => 'variable',

        T_CLASS                         => 'keyword',
        T_INTERFACE                     => 'keyword',
        T_EXTENDS                       => 'keyword',
        T_IMPLEMENTS                    => 'keyword',
        T_CATCH                         => 'keyword',
        T_THROW                         => 'keyword',
        T_TRY                           => 'keyword',
        T_NEW                           => 'keyword',
        T_FUNCTION                      => 'keyword',
        T_FOR                           => 'keyword',
        T_IF                            => 'keyword',
        T_ELSE                          => 'keyword',
        T_SWITCH                        => 'keyword',
        T_WHILE                         => 'keyword',
        T_FOREACH                       => 'keyword',
        T_RETURN                        => 'keyword',
        
        T_STATIC                        => 'modifier',
        T_ABSTRACT                      => 'modifier',
        T_PUBLIC                        => 'modifier',
        T_PRIVATE                       => 'modifier',
        T_PROTECTED                     => 'modifier',
        T_FINAL                         => 'modifier',
        
        T_DNUMBER                       => 'number',
        T_LNUMBER                       => 'number',
        T_CONSTANT_ENCAPSED_STRING      => 'string',
        
        T_COMMENT                       => 'comment',
        
        '{'                             => 'bracket',
        '}'                             => 'bracket',
        '('                             => 'bracket',
        ')'                             => 'bracket',
      );
      
      // Tokenize buffer
      $tokens= token_get_all('<?php '.trim($this->buffer, "\r\n").'?>');
      if (!is_array($tokens) || xp::errorAt(__FILE__, __LINE__ - 1)) {
        throw new FormatException('Cannot parse "'.$this->buffer.'"');
      }
      
      // Create HTML
      $current= NULL;
      $out= '';
      for ($i= 1, $s= sizeof($tokens)- 1; $i < $s; $i++) {
        $token= $tokens[$i];
        $class= isset($classes[$token[0]]) ? $classes[$token[0]] : 'default';
        
        // Handle annotations
        if (is_array($token) && T_COMMENT === $token[0] && '#' === $token[1][0]) {
          $class= 'annotation';
        }
        
        if ($current != $class) {
          $out.= '</span><span class="'.$class.'">';
          $current= $class;
        }

        $out.= strtr(htmlspecialchars(is_array($token) ? $token[1] : $token), array(
          "\n" => '<br/>',
          "\r" => ''
        ));
      }
      
      // Skip leading "</span>" (7)
      return '</p><code>'.substr($out, 7).($current ? '</span>' : '').'</code><p>';
    }
  }
?>
{'fraction_non_alphanumeric': 0.14894205504079658, 'fraction_numerical': 0.0027658691743880515, 'mean_word_length': 1.0314606741573034, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7481833', 'n_tokens_mistral': 2000, 'n_tokens_neox': 1918, 'n_words': 374}