<commit_before><?php
/* This file is part of the XP framework
 *
 * $Id$
 */
  require('lang.base.php');
  uses('io.File', 'util.text.PHPTokenizer', 'util.cmd.ParamString');
  
  $p= &new ParamString();
  if (2 != $p->count) {
    printf(
      "Strips comments and reduces whitespace on PHP sourcecode\n".
      "Usage: %s <filename>\n", 
      basename($p->value(0))
    );
    exit();
  }
  
  $t= &new PHPTokenizer();
  $f= &new File($p->value(1));
  try(); {
    $f->open(FILE_MODE_READ);
    $t->setTokens(token_get_all($f->read($f->size())));
    $f->close();
  } if (catch('Exception', $e)) {
    $e->printStackTrace();
    exit();
  }
  
  $tok= $t->getFirstToken();
  $str= '';
  do {
    printf("[%30s] '%s'\n", $t->getTokenName($tok[0]), $tok[1]);
    switch ($tok[0]) {
      case T_COMMENT: 
        // Strip comments
        break;
        
      case T_CONSTANT_ENCAPSED_STRING:
        // Copy "as-is"
        $str.= $tok[1];
        break;
        
      case T_WHITESPACE:
        // Reduce whitespace
        $str.= ' ';
        break;
      
      case T_OPEN_TAG:
        // Reduce whitespace after open tag
        $str.= '<?php';
        break;
        
      default:
        $str.= $tok[1];
    }
  } while ($tok= $t->getNextToken());
  echo $str;
?><commit_msg>- Remove unnecessary code<commit_after><?php
/* This file is part of the XP framework
 *
 * $Id$
 */
  require('lang.base.php');
  uses('io.File', 'util.text.PHPTokenizer', 'util.cmd.ParamString');
  
  $p= &new ParamString();
  if (2 != $p->count) {
    printf(
      "Strips comments and reduces whitespace on PHP sourcecode\n".
      "Usage: %s <filename>\n", 
      basename($p->value(0))
    );
    exit();
  }
  
  $t= &new PHPTokenizer();
  $f= &new File($p->value(1));
  try(); {
    $f->open(FILE_MODE_READ);
    $t->setTokens(token_get_all($f->read($f->size())));
    $f->close();
  } if (catch('Exception', $e)) {
    $e->printStackTrace();
    exit();
  }
  
  $tok= $t->getFirstToken();
  $str= '';
  do {
    printf("[%30s] '%s'\n", $t->getTokenName($tok[0]), $tok[1]);
    switch ($tok[0]) {
      case T_COMMENT: 
        // Strip comments
        break;
        
      case T_WHITESPACE:
        // Reduce whitespace
        $str.= ' ';
        break;
      
      case T_OPEN_TAG:
        // Reduce whitespace after open tag
        $str.= '<?php';
        break;
        
      default:
        $str.= $tok[1];
    }
  } while ($tok= $t->getNextToken());
  echo $str;
?>
{'fraction_non_alphanumeric': 0.20145337101332256, 'fraction_numerical': 0.007670569236980218, 'mean_word_length': 2.1052631578947367, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5249021', 'n_tokens_mistral': 922, 'n_tokens_neox': 914, 'n_words': 186}