<commit_before>var buffer = require('buffer');
var HTML5 = require('html5');

function Buffer() {
	this.data = new buffer.Buffer(4096);
	this.start = this.end = this.committed = 0;
	var eof;
	this.__defineSetter__('eof', function(f) {
		eof = f
	})
	this.__defineGetter__('eof', function() { return eof })
	this.eof = false;
}

exports.Buffer = Buffer;

Buffer.prototype = {
	slice: function() {
		return this.data.slice(this.start, this.end);
	},
	advance: function(amount) {
		this.start += amount;
		if(this.committed > this.data.length / 2) {
			// Sliiiide
			this.data.copy(this.data, 0, this.committed, this.end);
			this.end = this.end - this.committed;
			this.start = this.start - this.committed;
			this.committed = 0;
		}
	},
	matchWhile: function(re) {
		if(this.eof && this.end == this.start) return '';
		var r = new RegExp("^"+re+"+");
		if(m = r.exec(this.slice())) {
			if(!this.eof && m[0].length == this.end - this.start) throw('drain');
			this.advance(m[0].length);
			return m[0];
		} else {
			return '';
		}
	},
	matchUntil: function(re) {
		if(m = new RegExp(re + (this.eof ? "|$" : "")).exec(this.slice())) {
			var t = this.data.slice(this.start, this.start + m.index);
			this.advance(m.index);
			return t.toString();
		} else {
			if(this.eof) return '';
			throw('drain');
		}
	},
	append: function(data) {
		if(this.end + data.length <= this.data.length) {
			this.data.write(data, 'binary', this.end);
			this.end += data.length;
		} else {
			var temp = new buffer.Buffer((this.data.length + data.length) * 2);
			this.data.copy(temp, 0, this.committed, this.end);
			this.end -= this.committed;
			this.start -= this.committed;
			this.committed = 0;
			this.data = temp;
		}
	},
	shift: function(n) {
		if(this.eof && this.end == this.start) return HTML5.EOF;
		if(!this.eof && this.start + n >= this.end) throw('drain');
		d = this.data.slice(this.start, this.start + n).toString();
		this.advance(n);
		return d;
	},
	peek: function(n) {
		if(this.eof && this.end == this.start) return HTML5.EOF;
		if(!this.eof && this.start + n >= this.end) throw('drain');
		return this.data.slice(this.start, Math.min(this.start + n, this.end)).toString();
	},
	length: function() {
		return this.end - this.start;
	},<commit_msg>- actually append data on buffer resize<commit_after>var buffer = require('buffer');
var HTML5 = require('html5');

function Buffer() {
	this.data = new buffer.Buffer(4096);
	this.start = this.end = this.committed = 0;
	var eof;
	this.__defineSetter__('eof', function(f) {
		eof = f
	})
	this.__defineGetter__('eof', function() { return eof })
	this.eof = false;
}

exports.Buffer = Buffer;

Buffer.prototype = {
	slice: function() {
		return this.data.slice(this.start, this.end);
	},
	advance: function(amount) {
		this.start += amount;
		if(this.committed > this.data.length / 2) {
			// Sliiiide
			this.data.copy(this.data, 0, this.committed, this.end);
			this.end = this.end - this.committed;
			this.start = this.start - this.committed;
			this.committed = 0;
		}
	},
	matchWhile: function(re) {
		if(this.eof && this.end == this.start) return '';
		var r = new RegExp("^"+re+"+");
		if(m = r.exec(this.slice())) {
			if(!this.eof && m[0].length == this.end - this.start) throw('drain');
			this.advance(m[0].length);
			return m[0];
		} else {
			return '';
		}
	},
	matchUntil: function(re) {
		if(m = new RegExp(re + (this.eof ? "|$" : "")).exec(this.slice())) {
			var t = this.data.slice(this.start, this.start + m.index);
			this.advance(m.index);
			return t.toString();
		} else {
			if(this.eof) return '';
			throw('drain');
		}
	},
	append: function(data) {
		if(this.end + data.length <= this.data.length) {
			this.data.write(data, 'binary', this.end);
			this.end += data.length;
		} else {
			var temp = new buffer.Buffer((this.data.length + data.length) * 2);
			this.data.copy(temp, 0, this.committed, this.end);
			this.end -= this.committed;
			this.start -= this.committed;
			this.committed = 0;
			this.data = temp;
			this.append(data)
		}
	},
	shift: function(n) {
		if(this.eof && this.end == this.start) return HTML5.EOF;
		if(!this.eof && this.start + n >= this.end) throw('drain');
		d = this.data.slice(this.start, this.start + n).toString();
		this.advance(n);
		return d;
	},
	peek: function(n) {
		if(this.eof && this.end == this.start) return HTML5.EOF;
		if(!this.eof && this.start + n >= this.end) throw('drain');
		return this.data.slice(this.start, Math.min(this.start + n, this.end)).toString();
	},
	length: function() {
		return this.end - this.start;
	},
{'fraction_non_alphanumeric': 0.19653432770344373, 'fraction_numerical': 0.007896468523799079, 'mean_word_length': 3.3469971401334604, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7515229', 'n_tokens_mistral': 1902, 'n_tokens_neox': 1739, 'n_words': 339}