<commit_before>"use strict";

function Tokenizer()
{
	this.buffer = "";
	this.bufferLength = 0;
	this.cursor = 0;
	this.currChar = "";

	this.token = new Token();
};

Tokenizer.prototype = 
{
	setBuffer: function(buffer) 
	{
		this.buffer = buffer;
		this.bufferLength = buffer.length;
		this.cursor = 0;
		this.currChar = "";
	},

	nextToken: function()
	{
		this.token.type = 0;
		this.token.str = "";
		this.token.value = 0;

		this.nextChar();

		// Skip spaces
		while(isSpace(this.currChar)) {
			this.nextChar();
		}		

		// String
		if(isAlpha(this.currChar)) 
		{
			this.token.str += this.currChar;
			this.nextChar();
			while(isAlphaNum(this.currChar)) {
				this.token.str += this.currChar;
				this.nextChar();
			}
			this.cursor--;

			if(this.token.str === "var") {
				this.token.type = Token.Type.VAR;
			}
			else if(this.token.str === "return") {
				this.token.type = Token.Type.RETURN;
			}
			else if(this.token.str === "function") {
				this.token.type = Token.Type.FUNCTION;
			}
			else if(this.token.str === "true") {
				this.token.type = Token.Type.BOOL;
				this.token.value = 1;
			}
			else if(this.token.str === "false") {
				this.token.type = Token.Type.BOOL;			
			}
			else {
				this.token.type = Token.Type.NAME;
			}

			return this.token;
		}

		// Number
		if(isDigit(this.currChar)) <commit_msg>Added NaN as Number for tokeniser.<commit_after>"use strict";

function Tokenizer()
{
	this.buffer = "";
	this.bufferLength = 0;
	this.cursor = 0;
	this.currChar = "";

	this.token = new Token();
};

Tokenizer.prototype = 
{
	setBuffer: function(buffer) 
	{
		this.buffer = buffer;
		this.bufferLength = buffer.length;
		this.cursor = 0;
		this.currChar = "";
	},

	nextToken: function()
	{
		this.token.type = 0;
		this.token.str = "";
		this.token.value = 0;

		this.nextChar();

		// Skip spaces
		while(isSpace(this.currChar)) {
			this.nextChar();
		}		

		// String
		if(isAlpha(this.currChar)) 
		{
			this.token.str += this.currChar;
			this.nextChar();
			while(isAlphaNum(this.currChar)) {
				this.token.str += this.currChar;
				this.nextChar();
			}
			this.cursor--;

			if(this.token.str === "var") {
				this.token.type = Token.Type.VAR;
			}
			else if(this.token.str === "return") {
				this.token.type = Token.Type.RETURN;
			}
			else if(this.token.str === "function") {
				this.token.type = Token.Type.FUNCTION;
			}
			else if(this.token.str === "true") {
				this.token.type = Token.Type.BOOL;
				this.token.value = 1;
			}
			else if(this.token.str === "false") {
				this.token.type = Token.Type.BOOL;			
			}
			else if(this.token.str === "NaN") {
				this.token.type = Token.Type.NUMBER;
				this.token.value = NaN;
			}
			else {
				this.token.type = Token.Type.NAME;
			}

			return this.token;
		}

		// Number
		if(isDigit(this.currChar)) 
{'fraction_non_alphanumeric': 0.1763869132290185, 'fraction_numerical': 0.004267425320056899, 'mean_word_length': 2.6203346203346203, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2175626', 'n_tokens_mistral': 1258, 'n_tokens_neox': 1043, 'n_words': 168}