<commit_before>// based on http://kx.com/q/c/c.js
// uses Node.js Buffer instead of ArrayBuffer [michaelwittig]
// added support for decompression [GMelika] 

function decompress(compressedSize, b) {
	"use strict";
	var n = 0,
		r = 0,
		f = 0,
		s = 8,
		p = s,
		i = 0;
	var dst = new Buffer(compressedSize);
	var d = 12;
	var aa = new Int32Array(256);
	while (s < dst.length) {
		if (!i) {
			f = b[d++];
			i = 1;
		}
		if (f & i) {
			r = aa[b[d++]];
			dst[s++] = dst[r++];
			dst[s++] = dst[r++];
			n = 0xff & b[d++];
			for (var m = 0; m < n; m++) {
				dst[s + m] = dst[r + m];
			}
		} else {
			dst[s++] = b[d++];
		}
		while (p < (s - 1)) {
			aa[dst[p] ^ dst[p + 1]] = p++;
		}
		if (f & i) {
			p = (s += n);
		}
		i *= 2;
		if (i === 256) {
			i = 0;
		}
	}
	return dst;
}

function deserialize(b, nanos2date) {
	"use strict";
	var pos = 8,
		j2p32 = Math.pow(2, 32);

	var isCompressed = b[2] === 1;
	if (isCompressed) {
		var compressedSize = rInt32();
		b = decompress(compressedSize, b);
		pos = 8;
	}

	function rBool() {
		return rInt8() === 1;
	}
	function rChar() {
		return String.fromCharCode(rInt8());
	}
	function rInt8() {
		var val = b.readInt8(pos);
		pos += 1;
		return val;
	}
	function rNUInt8(n) {
		var val;
		if (n === 1) {
			val = b.readUInt8(pos);
		} else if (n === 2) {
			val = b.readUInt16LE(pos);
		} else if (n === 4) {
			val = b.readUInt32LE(pos);
		} else {
			throw new Error("only n = 1, 2 or 4 is supported");
		}
		pos += n;
		return val;
	}
	function rUInt8() {
		var val = b.readUInt8(pos);
		pos += 1;
		return val;
	}
	function rGuid() {
		var x = "0123456789abcdef",
			s = "";
		for (var i = 0; i < 16; i++) {
			var c = rUInt8();
			s += i == 4 || i === 6 || i === 8 || i === 10 ? "-" : "";
			s += x[c >> 4];
			s += x[c & 15];
		}
		return s;
	}
	function rInt16() {
		var h = rNUInt8(2);
		return h === -32768 ? NaN : h === -32767 ? -Infinity : h === 32767 ? Infinity : h;
	}
	function rInt32() {
		var i = rNUInt8(4);
		return i === -2147483648 ? NaN : i === -2147483647 ? -Infinity : i === 2147483647 ? Infinity : i;
	}
	function rInt64() { // closest number to 64 bit int...
		var y = rNUInt8(4);
		var x = rNUInt8(4);
		return x * j2p32 + (y >= 0 ? y : j2p32 + y);
	}
	function rFloat32() {
		var val = b.readFloatLE(pos);
		pos += 4;
		return val;
	}
	function rFloat64() {
		var val = b.readDoubleLE(pos);
		pos += 8;
		return val;
	}
	function rSymbol() {
		var i = pos, c, s = "";
		for (; (c = rInt8()) !== 0; s += String.fromCharCode(c));
		return s;
	}
	function rTimestamp() {
		if (nanos2date === false) {
			return 86400000000000 * (10957 + (rInt64() / 86400000000000));
		}<commit_msg>fixed #9<commit_after>// based on http://kx.com/q/c/c.js
// uses Node.js Buffer instead of ArrayBuffer [michaelwittig]
// added support for decompression [GMelika] 

function decompress(compressedSize, b) {
	"use strict";
	var n = 0,
		r = 0,
		f = 0,
		s = 8,
		p = s,
		i = 0;
	var dst = new Buffer(compressedSize);
	var d = 12;
	var aa = new Int32Array(256);
	while (s < dst.length) {
		if (!i) {
			f = b[d++];
			i = 1;
		}
		if (f & i) {
			r = aa[b[d++]];
			dst[s++] = dst[r++];
			dst[s++] = dst[r++];
			n = 0xff & b[d++];
			for (var m = 0; m < n; m++) {
				dst[s + m] = dst[r + m];
			}
		} else {
			dst[s++] = b[d++];
		}
		while (p < (s - 1)) {
			aa[dst[p] ^ dst[p + 1]] = p++;
		}
		if (f & i) {
			p = (s += n);
		}
		i *= 2;
		if (i === 256) {
			i = 0;
		}
	}
	return dst;
}

function deserialize(b, nanos2date) {
	"use strict";
	var pos = 8,
		j2p32 = Math.pow(2, 32);

	var isCompressed = b[2] === 1;
	if (isCompressed) {
		var compressedSize = rInt32();
		b = decompress(compressedSize, b);
		pos = 8;
	}

	function rBool() {
		return rInt8() === 1;
	}
	function rChar() {
		return String.fromCharCode(rInt8());
	}
	function rInt8() {
		var val = b.readInt8(pos);
		pos += 1;
		return val;
	}
	function rInt(n) {
		var val;
		if (n === 1) {
			val = b.readInt8(pos);
		} else if (n === 2) {
			val = b.readInt16LE(pos);
		} else if (n === 4) {
			val = b.readInt32LE(pos);
		} else {
			throw new Error("only n = 1, 2 or 4 is supported");
		}
		pos += n;
		return val;
	}
	function rUInt8() {
		var val = b.readUInt8(pos);
		pos += 1;
		return val;
	}
	function rGuid() {
		var x = "0123456789abcdef",
			s = "";
		for (var i = 0; i < 16; i++) {
			var c = rUInt8();
			s += i == 4 || i === 6 || i === 8 || i === 10 ? "-" : "";
			s += x[c >> 4];
			s += x[c & 15];
		}
		return s;
	}
	function rInt16() {
		var h = rInt(2);
		return h === -32768 ? NaN : h === -32767 ? -Infinity : h === 32767 ? Infinity : h;
	}
	function rInt32() {
		var i = rInt(4);
		return i === -2147483648 ? NaN : i === -2147483647 ? -Infinity : i === 2147483647 ? Infinity : i;
	}
	function rInt64() { // closest number to 64 bit int...
		var y = rInt(4);
		var x = rInt(4);
		return x * j2p32 + (y >= 0 ? y : j2p32 + y);
	}
	function rFloat32() {
		var val = b.readFloatLE(pos);
		pos += 4;
		return val;
	}
	function rFloat64() {
		var val = b.readDoubleLE(pos);
		pos += 8;
		return val;
	}
	function rSymbol() {
		var i = pos, c, s = "";
		for (; (c = rInt8()) !== 0; s += String.fromCharCode(c));
		return s;
	}
	function rTimestamp() {
		if (nanos2date === false) {
			return 86400000000000 * (10957 + (rInt64() / 86400000000000));
		}
{'fraction_non_alphanumeric': 0.20610974919856684, 'fraction_numerical': 0.07090326230435602, 'mean_word_length': 2.435233160621762, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3445342', 'n_tokens_mistral': 2793, 'n_tokens_neox': 2319, 'n_words': 643}