<commit_before><?php


class TiBuffer {
	private $text = "";	
	private $cursor;
	private $size;

	function loadText($text){
		$this->cursor = 0;
		$this->text = $text;
		$this->size = strlen($text);
	}

	function loadFile($filename){
		$this->cursor = 0;
		$this->text = file_get_contents($filename);
		$this->size = strlen($this->text);
	}

	function next(){
		if ( $this->cursor >= $this->size ){
			return '';
		}
		return $this->text[$this->cursor++];
	}

	function back(){
		if ( $this->cursor > 0 ){
			$this->cursor--;
		}
	}

	function isContinue(){
		if ( $this->cursor >= $this->size )
			return false;
		return true;
	}
}

class TiLex {
	private $lastsymbol;
	private $symbols = array();
	private $buffer;

	const  L_NONE = 0;
	const  L_CHAR = 1;
	const   L_INT = 2;
	const L_FLOAT = 3;
	const  L_ASPA = 4;
	const  L_SYMB = 5;


	function __construct (){
		$this->buffer = new TiBuffer();
		for ($i=0; $i<256; $i++){
			$this->symbols[$i] = self::L_NONE;
		}
		for ($i='a'; $i<='z'; $i++){
			$this->symbols[ord($i)] = self::L_CHAR;
		}
		for ($i='A'; $i<='Z'; $i++){
			$this->symbols[ord($i)] = self::L_CHAR;
		}
		for ($i='0'; $i<='9'; $i++){
			$this->symbols[ord($i)] = self::L_INT;
		}

		$this->symbols[ord(' ')]  = self::L_NONE;
		$this->symbols[ord("\t")] = self::L_NONE;
		$this->symbols[ord('=')]  = self::L_SYMB;
		$this->symbols[ord(':')]  = self::L_CHAR;
		$this->symbols[ord(';')]  = self::L_SYMB;
		$this->symbols[ord('.')]  = self::L_SYMB;
		$this->symbols[ord(',')]  = self::L_SYMB;
		$this->symbols[ord('(')]  = self::L_ASPA;
		$this->symbols[ord(')')]  = self::L_ASPA;
		$this->symbols[ord('{')]  = self::L_SYMB;
		$this->symbols[ord('}')]  = self::L_SYMB;
		$this->symbols[ord('[')]  = self::L_SYMB;
		$this->symbols[ord(']')]  = self::L_SYMB;
		$this->symbols[ord('\'')] = self::L_ASPA;
		$this->symbols[ord('"')]  = self::L_ASPA;
		$this->symbols[ord('-')]  = self::L_INT;
		$this->symbols[ord('_')]  = self::L_CHAR;
		$this->symbols[ord('@')]  = self::L_CHAR;
		$this->symbols[ord('$')]  = self::L_CHAR;
		$this->symbols[ord('&')]  = self::L_CHAR;
		$this->symbols[ord('#')]  = self::L_CHAR;
		$this->lastsymbol = 0;
	}


	function loadFile($filename){
		$this->buffer->loadFile($filename);
	}

	function loadText($text){
		$this->buffer->loadText($text);
	}

	function next(){
		$out["token"] = "";
		$out["type"] = self::L_NONE;
		$c = '';
		$aspa=0;
		$type = $this->symbols[ord($this->lastsymbol)];
		if ( $type == self::L_SYMB ){
			$this->buffer->back();
		} else if ( $type == self::L_CHAR ){
			$this->buffer->back();
		}
		$this->lastsymbol = 0;

		while ( $this->buffer->isContinue() ){
			$c = $this->buffer->next();
			$type = $this->symbols[ord($c)];
			if ( $type != self::L_NONE ){
				$aspa = $c;
				break;
			}
		}
		if ( $type == self::L_SYMB ){
			$out['token'] = $c;
			$out['type']  = self::L_SYMB;
			return $out;
		} else if ( $type == self::L_ASPA ){
			if ( $c == '(' )
				$aspa = ')';
			$special = false;
			while ( $this->buffer->isContinue() ){
				$c = $this->buffer->next();
				if ( $special ){
					if ( $c == 'n' )
						$out['token'] .= '\n';
					else if ( $c == 't' )
						$out['token'] .= '\t';
					else if ( $c == '\'' )
						$out['token'] .= '\'';
					else if ( $c == '\"' )
						$out['token'] .= '\"';
					else if ( $c == '\\' )
						$out['token'] .= '\\';
					$special = false;
				} else if ( $c == '\\' ){
					$special = true;
				} else if ( $c == $aspa ){
					break;
				} else
					$out['token'] .= $c;
			}
			$out['type'] = self::L_CHAR;
		} else if ( $type == self::L_CHAR ){
			$out['token'] = $c;
			while ( $this->buffer->isContinue() ){
				$c = $this->buffer->next();
				$type = $this->symbols[ord($c)];
				if ( $type == self::L_CHAR || $type == self::L_INT ){
					$out['token'] .= $c;
				} else {
					$this->lastsymbol = $c;
					break;
				}
			}
			$out['type'] = self::L_CHAR;
		} else if ( $type == self::L_INT ){
			$out['token'] = $c;
			$out['type'] = self::L_INT;
			while ( $this->buffer->isContinue() ){
				$c = $this->buffer->next();
				$type = $this->symbols[ord($c)];
				if ( $c == '.' || $c == 'e' ){
					$out['type'] = self::L_FLOAT;
				} else if ( $type != self::L_INT ){
					$this->lastsymbol = $c;
					break;
				}
				$out['token'] .= $c;
			}
		}
		return $out;
	}

	function isContinue(){
		return $this->buffer->isContinue();
	}
}







class TiParser {
	private $mem_i=0;
	private $lex;
	private $translate = array();
	private $memory = array();
	private $PARSER_ACTION;<commit_msg>PHP: arrumado bug com o caracter ',' para o parser de numeros flutuantes<commit_after><?php


class TiBuffer {
	private $text = "";	
	private $cursor;
	private $size;

	function loadText($text){
		$this->cursor = 0;
		$this->text = $text;
		$this->size = strlen($text);
	}

	function loadFile($filename){
		$this->cursor = 0;
		$this->text = file_get_contents($filename);
		$this->size = strlen($this->text);
	}

	function next(){
		if ( $this->cursor >= $this->size ){
			return '';
		}
		return $this->text[$this->cursor++];
	}

	function back(){
		if ( $this->cursor > 0 ){
			$this->cursor--;
		}
	}

	function isContinue(){
		if ( $this->cursor >= $this->size )
			return false;
		return true;
	}
}

class TiLex {
	private $lastsymbol;
	private $symbols = array();
	private $buffer;

	const  L_NONE = 0;
	const  L_CHAR = 1;
	const   L_INT = 2;
	const L_FLOAT = 3;
	const  L_ASPA = 4;
	const  L_SYMB = 5;


	function __construct (){
		$this->buffer = new TiBuffer();
		for ($i=0; $i<256; $i++){
			$this->symbols[$i] = self::L_NONE;
		}
		for ($i='a'; $i<='z'; $i++){
			$this->symbols[ord($i)] = self::L_CHAR;
		}
		for ($i='A'; $i<='Z'; $i++){
			$this->symbols[ord($i)] = self::L_CHAR;
		}
		for ($i='0'; $i<='9'; $i++){
			$this->symbols[ord($i)] = self::L_INT;
		}

		$this->symbols[ord(' ')]  = self::L_NONE;
		$this->symbols[ord("\t")] = self::L_NONE;
		$this->symbols[ord('=')]  = self::L_SYMB;
		$this->symbols[ord(':')]  = self::L_CHAR;
		$this->symbols[ord(';')]  = self::L_SYMB;
		$this->symbols[ord('.')]  = self::L_SYMB;
		$this->symbols[ord(',')]  = self::L_SYMB;
		$this->symbols[ord('(')]  = self::L_ASPA;
		$this->symbols[ord(')')]  = self::L_ASPA;
		$this->symbols[ord('{')]  = self::L_SYMB;
		$this->symbols[ord('}')]  = self::L_SYMB;
		$this->symbols[ord('[')]  = self::L_SYMB;
		$this->symbols[ord(']')]  = self::L_SYMB;
		$this->symbols[ord('\'')] = self::L_ASPA;
		$this->symbols[ord('"')]  = self::L_ASPA;
		$this->symbols[ord('-')]  = self::L_INT;
		$this->symbols[ord('_')]  = self::L_CHAR;
		$this->symbols[ord('@')]  = self::L_CHAR;
		$this->symbols[ord('$')]  = self::L_CHAR;
		$this->symbols[ord('&')]  = self::L_CHAR;
		$this->symbols[ord('#')]  = self::L_CHAR;
		$this->lastsymbol = 0;
	}


	function loadFile($filename){
		$this->buffer->loadFile($filename);
	}

	function loadText($text){
		$this->buffer->loadText($text);
	}

	function next(){
		$out["token"] = "";
		$out["type"] = self::L_NONE;
		$c = '';
		$aspa=0;
		$type = $this->symbols[ord($this->lastsymbol)];
		if ( $type == self::L_SYMB ){
			$this->buffer->back();
		} else if ( $type == self::L_CHAR ){
			$this->buffer->back();
		}
		$this->lastsymbol = 0;

		while ( $this->buffer->isContinue() ){
			$c = $this->buffer->next();
			$type = $this->symbols[ord($c)];
			if ( $type != self::L_NONE ){
				$aspa = $c;
				break;
			}
		}
		if ( $type == self::L_SYMB ){
			$out['token'] = $c;
			$out['type']  = self::L_SYMB;
			return $out;
		} else if ( $type == self::L_ASPA ){
			if ( $c == '(' )
				$aspa = ')';
			$special = false;
			while ( $this->buffer->isContinue() ){
				$c = $this->buffer->next();
				if ( $special ){
					if ( $c == 'n' )
						$out['token'] .= '\n';
					else if ( $c == 't' )
						$out['token'] .= '\t';
					else if ( $c == '\'' )
						$out['token'] .= '\'';
					else if ( $c == '\"' )
						$out['token'] .= '\"';
					else if ( $c == '\\' )
						$out['token'] .= '\\';
					$special = false;
				} else if ( $c == '\\' ){
					$special = true;
				} else if ( $c == $aspa ){
					break;
				} else
					$out['token'] .= $c;
			}
			$out['type'] = self::L_CHAR;
		} else if ( $type == self::L_CHAR ){
			$out['token'] = $c;
			while ( $this->buffer->isContinue() ){
				$c = $this->buffer->next();
				$type = $this->symbols[ord($c)];
				if ( $type == self::L_CHAR || $type == self::L_INT ){
					$out['token'] .= $c;
				} else {
					$this->lastsymbol = $c;
					break;
				}
			}
			$out['type'] = self::L_CHAR;
		} else if ( $type == self::L_INT ){
			$out['token'] = $c;
			$out['type'] = self::L_INT;
			while ( $this->buffer->isContinue() ){
				$c = $this->buffer->next();
				$type = $this->symbols[ord($c)];
				if ( $c == '.' || $c == 'e' ){
					$out['type'] = self::L_FLOAT;
				} else if ( $c == ',' ){
					$out['type'] = self::L_FLOAT;
					$c = '.';					
				} else if ( $type != self::L_INT ){
					$this->lastsymbol = $c;
					break;
				}
				$out['token'] .= $c;
			}
		}
		return $out;
	}

	function isContinue(){
		return $this->buffer->isContinue();
	}
}







class TiParser {
	private $mem_i=0;
	private $lex;
	private $translate = array();
	private $memory = array();
	private $PARSER_ACTION;
{'fraction_non_alphanumeric': 0.26205770277626567, 'fraction_numerical': 0.0041371801850843765, 'mean_word_length': 3.0096027935399388, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 161, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7420811', 'n_tokens_mistral': 4438, 'n_tokens_neox': 3809, 'n_words': 664}