<commit_before>var fs = require('fs'),
	expat = require('node-expat'),
	path = require('path'),
	unzip = require('unzip'),
	util = require('util'),
	events = require('events');

/*

 */

var consts = {
	/*
	 xlsx build in nr formats
	 */
	fmts: {
		0: null,//General
		1: '0',
		2: '0.00',
		3: '#,##0',
		4: '#,##0.00',

		9: '0%',
		10: '0.00%',
		11: '0.00E+00',
		12: '# ?/?',
		13: '# ??/??',
		14: 'mm-dd-yy',
		15: 'd-mmm-yy',
		16: 'd-mmm',
		17: 'mmm-yy',
		18: 'h:mm AM/PM',
		19: 'h:mm:ss AM/PM',
		20: 'h:mm',
		21: 'h:mm:ss',
		22: 'm/d/yy h:mm',

		37: '#,##0 ;(#,##0)',
		38: '#,##0 ;[Red](#,##0)',
		39: '#,##0.00;(#,##0.00)',
		40: '#,##0.00;[Red](#,##0.00)',

		45: 'mm:ss',
		46: '[h]:mm:ss',
		47: 'mmss.0',
		48: '##0.0E+0',
		49: '@'
	},
	alphabet: 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
};


/*

 */

var _utils = {
	/*
	 converts a raw xlsx-date to js date
	 */
	xlsx_date: function (value, date1904) {
		var date = Math.floor(value),
			time = Math.round(86400 * (value - date)),
			d;
		if (date1904) date += 1462;
		// Open XML stores dates as the number of days from 1 Jan 1900. Well, skipping the incorrect 29 Feb 1900 as a valid day.
		if (date === 60) {
			d = new Date(1900, 1, 29);
		} else {
			if (date > 60) --date;
			/* 1 = Jan 1 1900 */
			d = new Date(1900, 0, 1, 0, 0, 0);
			d.setDate(d.getDate() + date - 1);
		}
		d.setSeconds(time % 60);
		time = Math.floor(time / 60);
		d.setMinutes(time % 60);
		time = Math.floor(time / 60);
		d.setHours(time);
		return d;
	},

	/*
	 converts a column index to chars e.g. 1 -> A
	 */
	numAlpha: function (i) {
		var t = Math.floor(i / 26) - 1;
		return (t > -1 ? this.numAlpha(t) : '') + consts.alphabet.charAt(i % 26);
	},

	/*
	 converts a chars to column index e.g. A -> 1
	 */
	alphaNum: function (name) {
		var result = 0;
		var multiplier = 1;
		for (var i = name.length - 1; i >= 0; i--) {
			var value = ((name[i].charCodeAt(0) - "A".charCodeAt(0)) + 1);
			result = result + value * multiplier;
			multiplier = multiplier * 26;
		}
		return (result - 1);
	},

	splitFormats: function (s) {
		// http://office.microsoft.com/en-gb/excel-help/create-or-delete-a-custom-number-format-HP005199500.aspx?redir=0
		//    _-* #,##0\ _€_-;\-* #,##0\ _€_-;_-* "-"??\ _€_-;_-@_-
		//    positiv value ; negativ value ; zero; string

		var fmts = s.split(/(?!\\);/);
		var nr = 0;
		var lastff = {t: 'x'};
		var result = [];
		for (var i = 0; i < fmts.length; i++) {
			var ff = this.parseFmtType(fmts[i]);
			ff = (ff.t == 'l' ? lastff : ff);
			lastff = ff;
			var format = {fmt: fmts[i], fmt_type: ff.t};
			if (ff.f)
				format.digits = ff.f;
			result.push(format);
			nr++;
		}
		return result;
	},

	parseFmtType: function (fmt) {
		var containsOnlyChars = function (value, chars) {
			for (var i = 0; i < value.length; i++) {
				if (chars.indexOf(value[i]) < 0) {
					return false;
				}
			}
			return (value.length > 0);
		};
		//messy hack for extracting some infos from the number format (type and float-digits}
		var s = fmt;
		var b = '';
		while (s.length > 0) {
			var c = s[0];
			s = s.slice(1);
			if ((c == '_') || (c == '\\') || (c == '*'))
				s = s.slice(1);
			else if (c == '[')
				s = s.slice(s.indexOf(']') + 1);
			else if (c == '"')
				s = s.slice(s.indexOf('"') + 1);
			else if ((c == '(')
				|| (c == ')')
				) {

			} else
				b += c;
		}
		b = b.replace(/#/g, '0');
		b = b.replace(/%/g, '');
		// deal with thousands separator 12000 -> 12 -> formatCode	"#,"
		var sp = b.split(',');
		b = sp[sp.length - 1];
		if (!isNaN(b)) {
			if (b.indexOf('.') >= 0) {
				var di = sp[sp.length - 1].split('.')[1].trim().length;
				if (b.indexOf('E+') >= 0)
					di += 14;
				return {t: 'f', f: di};
			} else
				return {t: 'i'};
		} else if (b == '@')
			return {t: 's'};
		//"-"??    zero value
		if (b == '??')
			return {t: 'l'}; // last fmt should by used
		sp = b.split(' ');
		if (sp.length > 1) {
			//test # ??/??
			if (containsOnlyChars(sp[sp.length - 1], '?/')) {
				// '# ?/?', '# ??/??',
				var digits = sp[sp.length - 1].split('/')[0].trim().length + 1;
				return {t: 'f', f: digits};
			}
		}
		//date format?
		if (containsOnlyChars(b, 'tmdyhseAPTMH:/-.0 ')) {
			return {t: 'd'};
		}
		console.log('bef:', fmt, 'aft:', b);
		console.log('unknown', b);
		return {t: 'x'};
	}

};

/*

 */

function Cell() {
	this.val = null;
	this.typ = null;
	this.col = -1;
	this.fmt = null;
	this.raw = null;
}
Cell.prototype.getFormat = function (options) {
	switch (options.format) {
		case 'json':
			return this.toJson();
		case 'array':
			return this.val;
		case 'obj':
			return this;
		case 'tsv':
		default:
			return this.toTSV(options.tsv_float_comma);
	}
};
Cell.prototype.toTSV = function (tsv_float_comma) {
	if (this.val == null)
		return '';
	if (util.isDate(this.val))
		return this.val.toISOString();
	if (tsv_float_comma && (typeof this.val == 'number'))
		return this.val.toString().replace('.', ',');
	return this.val.toString();
};
Cell.prototype.toJson = function () {
	return JSON.stringify(this.val);
};
Cell.prototype.getEffectiveNumFormat = function () {
	if ((!this.fmt) || (this.fmt.fmts.length == 0))
		return null;<commit_msg>Handle cells with strings that break proper TSV files (tabs, newlines, and quotes).<commit_after>var fs = require('fs'),
	expat = require('node-expat'),
	path = require('path'),
	unzip = require('unzip'),
	util = require('util'),
	events = require('events');

/*

 */

var consts = {
	/*
	 xlsx build in nr formats
	 */
	fmts: {
		0: null,//General
		1: '0',
		2: '0.00',
		3: '#,##0',
		4: '#,##0.00',

		9: '0%',
		10: '0.00%',
		11: '0.00E+00',
		12: '# ?/?',
		13: '# ??/??',
		14: 'mm-dd-yy',
		15: 'd-mmm-yy',
		16: 'd-mmm',
		17: 'mmm-yy',
		18: 'h:mm AM/PM',
		19: 'h:mm:ss AM/PM',
		20: 'h:mm',
		21: 'h:mm:ss',
		22: 'm/d/yy h:mm',

		37: '#,##0 ;(#,##0)',
		38: '#,##0 ;[Red](#,##0)',
		39: '#,##0.00;(#,##0.00)',
		40: '#,##0.00;[Red](#,##0.00)',

		45: 'mm:ss',
		46: '[h]:mm:ss',
		47: 'mmss.0',
		48: '##0.0E+0',
		49: '@'
	},
	alphabet: 'ABCDEF<KEY>'
};


/*

 */

var _utils = {
	/*
	 converts a raw xlsx-date to js date
	 */
	xlsx_date: function (value, date1904) {
		var date = Math.floor(value),
			time = Math.round(86400 * (value - date)),
			d;
		if (date1904) date += 1462;
		// Open XML stores dates as the number of days from 1 Jan 1900. Well, skipping the incorrect 29 Feb 1900 as a valid day.
		if (date === 60) {
			d = new Date(1900, 1, 29);
		} else {
			if (date > 60) --date;
			/* 1 = Jan 1 1900 */
			d = new Date(1900, 0, 1, 0, 0, 0);
			d.setDate(d.getDate() + date - 1);
		}
		d.setSeconds(time % 60);
		time = Math.floor(time / 60);
		d.setMinutes(time % 60);
		time = Math.floor(time / 60);
		d.setHours(time);
		return d;
	},

	/*
	 converts a column index to chars e.g. 1 -> A
	 */
	numAlpha: function (i) {
		var t = Math.floor(i / 26) - 1;
		return (t > -1 ? this.numAlpha(t) : '') + consts.alphabet.charAt(i % 26);
	},

	/*
	 converts a chars to column index e.g. A -> 1
	 */
	alphaNum: function (name) {
		var result = 0;
		var multiplier = 1;
		for (var i = name.length - 1; i >= 0; i--) {
			var value = ((name[i].charCodeAt(0) - "A".charCodeAt(0)) + 1);
			result = result + value * multiplier;
			multiplier = multiplier * 26;
		}
		return (result - 1);
	},

	splitFormats: function (s) {
		// http://office.microsoft.com/en-gb/excel-help/create-or-delete-a-custom-number-format-HP005199500.aspx?redir=0
		//    _-* #,##0\ _€_-;\-* #,##0\ _€_-;_-* "-"??\ _€_-;_-@_-
		//    positiv value ; negativ value ; zero; string

		var fmts = s.split(/(?!\\);/);
		var nr = 0;
		var lastff = {t: 'x'};
		var result = [];
		for (var i = 0; i < fmts.length; i++) {
			var ff = this.parseFmtType(fmts[i]);
			ff = (ff.t == 'l' ? lastff : ff);
			lastff = ff;
			var format = {fmt: fmts[i], fmt_type: ff.t};
			if (ff.f)
				format.digits = ff.f;
			result.push(format);
			nr++;
		}
		return result;
	},

	parseFmtType: function (fmt) {
		var containsOnlyChars = function (value, chars) {
			for (var i = 0; i < value.length; i++) {
				if (chars.indexOf(value[i]) < 0) {
					return false;
				}
			}
			return (value.length > 0);
		};
		//messy hack for extracting some infos from the number format (type and float-digits}
		var s = fmt;
		var b = '';
		while (s.length > 0) {
			var c = s[0];
			s = s.slice(1);
			if ((c == '_') || (c == '\\') || (c == '*'))
				s = s.slice(1);
			else if (c == '[')
				s = s.slice(s.indexOf(']') + 1);
			else if (c == '"')
				s = s.slice(s.indexOf('"') + 1);
			else if ((c == '(')
				|| (c == ')')
				) {

			} else
				b += c;
		}
		b = b.replace(/#/g, '0');
		b = b.replace(/%/g, '');
		// deal with thousands separator 12000 -> 12 -> formatCode	"#,"
		var sp = b.split(',');
		b = sp[sp.length - 1];
		if (!isNaN(b)) {
			if (b.indexOf('.') >= 0) {
				var di = sp[sp.length - 1].split('.')[1].trim().length;
				if (b.indexOf('E+') >= 0)
					di += 14;
				return {t: 'f', f: di};
			} else
				return {t: 'i'};
		} else if (b == '@')
			return {t: 's'};
		//"-"??    zero value
		if (b == '??')
			return {t: 'l'}; // last fmt should by used
		sp = b.split(' ');
		if (sp.length > 1) {
			//test # ??/??
			if (containsOnlyChars(sp[sp.length - 1], '?/')) {
				// '# ?/?', '# ??/??',
				var digits = sp[sp.length - 1].split('/')[0].trim().length + 1;
				return {t: 'f', f: digits};
			}
		}
		//date format?
		if (containsOnlyChars(b, 'tmdyhseAPTMH:/-.0 ')) {
			return {t: 'd'};
		}
		console.log('bef:', fmt, 'aft:', b);
		console.log('unknown', b);
		return {t: 'x'};
	}

};

/*

 */

function Cell() {
	this.val = null;
	this.typ = null;
	this.col = -1;
	this.fmt = null;
	this.raw = null;
}
Cell.prototype.getFormat = function (options) {
	switch (options.format) {
		case 'json':
			return this.toJson();
		case 'array':
			return this.val;
		case 'obj':
			return this;
		case 'tsv':
		default:
			return this.toTSV(options.tsv_float_comma);
	}
};
Cell.prototype.toTSV = function (tsv_float_comma) {
	var val;
	if (this.val == null)
		return '';
	if (util.isDate(this.val))
		return this.val.toISOString();
	if (tsv_float_comma && (typeof this.val == 'number'))
		return this.val.toString().replace('.', ',');
	val = this.val.toString();
	if (val.indexOf("\"") > -1 || val.indexOf("\n") > -1 || val.indexOf("\t") > -1) {
		val = "\"" + val.replace( /\"/g, "\"\"") + "\"";
	}
	return val;
};
Cell.prototype.toJson = function () {
	return JSON.stringify(this.val);
};
Cell.prototype.getEffectiveNumFormat = function () {
	if ((!this.fmt) || (this.fmt.fmts.length == 0))
		return null;
{'fraction_non_alphanumeric': 0.24008726996774807, 'fraction_numerical': 0.04088408271675204, 'mean_word_length': 2.5367326400536734, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6394983', 'n_tokens_mistral': 5275, 'n_tokens_neox': 4442, 'n_words': 1081}