<commit_before>
/**
 * Takes the contents of a file and generates a statistics
 * object with information about the lines.
*/
exports = module.exports = function(contents, options, callback) {
	if(!callback && typeof options === 'function') {
		callback = options;
		options = {};
	}

	// Set defauls if no options are passed
	if(!options.singleLineComment) options.singleLineComment = '//';
	if(!options.blockCommentOpen) options.blockCommentOpen = '/*';
	if(!options.blockCommentClose) options.blockCommentClose = '*/';

	var statistics = {
		'total': 0,
		'source': 0,
		'singleLineComments': 0,
		'blockComments': 0,
		'empty': 0
	};

	var lines = contents.split(options.lineSeparator || '\r\n');
	var line, blockCommentOpen;
	for(var i = 0; i < lines.length; i++) {
		statistics.total++;

		line = lines[i].trim();<commit_msg>Improve options handling<commit_after>
/**
 * Takes the contents of a file and generates a statistics
 * object with information about the lines.
*/
exports = module.exports = function(contents, options, callback) {
	if(!callback && typeof options === 'function') {
		callback = options;
		options = {};
	}
	if(!options) options = {};

	// Set defauls if no options are passed
	if(!options.singleLineComment) options.singleLineComment = '//';
	if(!options.blockCommentOpen) options.blockCommentOpen = '/*';
	if(!options.blockCommentClose) options.blockCommentClose = '*/';

	var statistics = {
		'total': 0,
		'source': 0,
		'singleLineComments': 0,
		'blockComments': 0,
		'empty': 0
	};

	var lines = contents.split(options.lineSeparator || '\r\n');
	var line, blockCommentOpen;
	for(var i = 0; i < lines.length; i++) {
		statistics.total++;

		line = lines[i].trim();
{'fraction_non_alphanumeric': 0.15696649029982362, 'fraction_numerical': 0.007054673721340388, 'mean_word_length': 4.285714285714286, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '860946', 'n_tokens_mistral': 566, 'n_tokens_neox': 524, 'n_words': 143}