<commit_before><?php

require __DIR__ . '/lexer_1.php';

class lexer
{
	private $toks = [];

	function __construct($path)
	{
		$s = new lexer_1(file_get_contents($path));
		$this->toks = [];
		while (true) {
			$tok = $s->get();
			if (!$tok) break;
			if ($tok->type == 'error') {
				throw new Exception("$tok->content at $tok->pos");
			}
			$this->toks[] = $tok;
		}
	}

	function ended()
	{
		return count($this->toks) == 0;
	}

	function get()
	{
		while (true) {
			if (empty($this->toks)) {
				// var_dump('get', 'null');
				return null;
			}
			$tok = array_shift($this->toks);
			// var_dump('get', $tok);
			if ($tok->type != 'comment') return $tok;
		}
	}

	/*
	 * Pushes a token back to the stream.
	 */
	function unget($t)
	{
		array_unshift($this->toks, $t);
	}

	/*
	 * Returns next token without removing it from the stream.
	 * Returns null if there is no next token.<commit_msg>cache lexer results<commit_after><?php

require __DIR__ . '/lexer_1.php';

function debmsg($m)
{
	echo "$m\n";
}

function read($path)
{
	static $cache = [];
	if (!isset($cache[$path])) {
		$toks = [];
		$s = new lexer_1(file_get_contents($path));
		debmsg('parsing ' . $path);
		while (true) {
			$tok = $s->get();
			if (!$tok) break;
			if ($tok->type == 'error') {
				throw new Exception("$tok->content at $tok->pos");
			}
			$toks[] = $tok;
		}
		$cache[$path] = $toks;
	}
	return $cache[$path];
}

class lexer
{
	private $toks = [];

	function __construct($path)
	{
		$this->toks = read($path);
	}

	function ended()
	{
		return count($this->toks) == 0;
	}

	function get()
	{
		while (true) {
			if (empty($this->toks)) {
				// var_dump('get', 'null');
				return null;
			}
			$tok = array_shift($this->toks);
			// var_dump('get', $tok);
			if ($tok->type != 'comment') return $tok;
		}
	}

	/*
	 * Pushes a token back to the stream.
	 */
	function unget($t)
	{
		array_unshift($this->toks, $t);
	}

	/*
	 * Returns next token without removing it from the stream.
	 * Returns null if there is no next token.
{'fraction_non_alphanumeric': 0.20218037661050545, 'fraction_numerical': 0.002973240832507433, 'mean_word_length': 2.9666011787819255, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4781259', 'n_tokens_mistral': 880, 'n_tokens_neox': 780, 'n_words': 185}