<commit_before>require('core-upgrade');
var HTML5 = require('html5');
var events = require('events');
var Buffer = require('html5/buffer').Buffer;
var Models = HTML5.Models;

function keys(h) {
	var r = [];
	for(var k in h) {
		r.push(k);
	}
	return r;
}

HTML5.Tokenizer = t = function HTML5Tokenizer(input, ready) {
	HTML5.debug('tokenizer', 'initializing tokenizer');
	this.content_model = Models.PCDATA;
	var state;
	var buffer = this.buffer = new Buffer();
	this.__defineSetter__('state', function(newstate) {
		HTML5.debug('tokenizer.state', newstate)
		state = newstate;
		buffer.commit();
	});
	this.state = 'data_state';
	this.escapeFlag = false;
	this.lastFourChars = '';
	this.current_token = null;

	if(typeof(input) == 'string') {
		var source = new events.EventEmitter();
	} else {
		var source = input;
	}
	
	this.commit = function() {
		buffer.commit();
	};

	if(ready) ready(this);

	var tokenizer = this;
	source.addListener('data', function(data) {
		if(typeof data !== 'string') data = data.toString();
		buffer.append(data);
		try {
			while(buffer.length() > 0) tokenizer[state](buffer);
		} catch(e) {
		       if(e != 'drain') {
				HTML5.debug('tokenizer.error', "Got an exception while tokenizing -- " + e );
			       throw(e);
		       } else {
			       buffer.undo();
		       }
		}
	});
	source.addListener('end', function() {
		buffer.eof = true;
		while(tokenizer[state](buffer));
	});

	if(source != input) {
		source.emit('data', input);
		source.emit('end');
	}
}

t.prototype = new events.EventEmitter;

t.prototype.emitToken = function(tok) { 
	HTML5.debug('tokenizer.emitToken', tok);
	this.emit('token', tok);
}

t.prototype.consume_entity = function(buffer, from_attr) {
	HTML5.debug('tokenizer', 'consume_entity');
	var char = null;
	var chars = buffer.shift(1);
	// FIXME: no EOF handling in this function
	if(chars.match(HTML5.SPACE_CHARACTERS) || chars == '<' || chars == '&') {
		buffer.unget(chars);
	} else if(chars.match(/^#/)) { // Maybe a numeric entity
		chars = chars + buffer.shift(2);
		if(chars[1].toLowerCase() == 'x' && HTML5.HEX_DIGITS_R.test(chars[2])) {
			// Hex entity
			buffer.unget(chars[2]);
			char = this.consume_numeric_entity(buffer, true);
		} else if(HTML5.DIGITS.test(chars[1])) {
			// Decimal entity
			buffer.unget(chars.slice(1));
			char = this.consume_numeric_entity(buffer, false);
		} else {<commit_msg>- fix numeric entities (Decimal)<commit_after>require('core-upgrade');
var HTML5 = require('html5');
var events = require('events');
var Buffer = require('html5/buffer').Buffer;
var Models = HTML5.Models;

function keys(h) {
	var r = [];
	for(var k in h) {
		r.push(k);
	}
	return r;
}

HTML5.Tokenizer = t = function HTML5Tokenizer(input, ready) {
	HTML5.debug('tokenizer', 'initializing tokenizer');
	this.content_model = Models.PCDATA;
	var state;
	var buffer = this.buffer = new Buffer();
	this.__defineSetter__('state', function(newstate) {
		HTML5.debug('tokenizer.state', newstate)
		state = newstate;
		buffer.commit();
	});
	this.state = 'data_state';
	this.escapeFlag = false;
	this.lastFourChars = '';
	this.current_token = null;

	if(typeof(input) == 'string') {
		var source = new events.EventEmitter();
	} else {
		var source = input;
	}
	
	this.commit = function() {
		buffer.commit();
	};

	if(ready) ready(this);

	var tokenizer = this;
	source.addListener('data', function(data) {
		if(typeof data !== 'string') data = data.toString();
		buffer.append(data);
		try {
			while(buffer.length() > 0) tokenizer[state](buffer);
		} catch(e) {
		       if(e != 'drain') {
				HTML5.debug('tokenizer.error', "Got an exception while tokenizing -- " + e );
			       throw(e);
		       } else {
			       buffer.undo();
		       }
		}
	});
	source.addListener('end', function() {
		buffer.eof = true;
		while(tokenizer[state](buffer));
	});

	if(source != input) {
		source.emit('data', input);
		source.emit('end');
	}
}

t.prototype = new events.EventEmitter;

t.prototype.emitToken = function(tok) { 
	HTML5.debug('tokenizer.emitToken', tok);
	this.emit('token', tok);
}

t.prototype.consume_entity = function(buffer, from_attr) {
	HTML5.debug('tokenizer', 'consume_entity');
	var char = null;
	var chars = buffer.shift(1);
	// FIXME: no EOF handling in this function
	if(chars.match(HTML5.SPACE_CHARACTERS) || chars == '<' || chars == '&') {
		buffer.unget(chars);
	} else if(chars.match(/^#/)) { // Maybe a numeric entity
		chars = chars + buffer.shift(2);
		if(chars[1].toLowerCase() == 'x' && HTML5.HEX_DIGITS_R.test(chars[2])) {
			// Hex entity
			buffer.unget(chars[2]);
			char = this.consume_numeric_entity(buffer, true);
		} else if(HTML5.DIGITS_R.test(chars[1])) {
			// Decimal entity
			buffer.unget(chars.slice(1));
			char = this.consume_numeric_entity(buffer, false);
		} else {
{'fraction_non_alphanumeric': 0.17804216238781048, 'fraction_numerical': 0.00918388645376748, 'mean_word_length': 3.879837067209776, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1965161', 'n_tokens_mistral': 1826, 'n_tokens_neox': 1688, 'n_words': 336}