<commit_before>var sys = require("sys");
var fs = require("fs");
var path = require("path");
var Buffer = require('buffer').Buffer;

var defaultFilters = {
	escape: function(data, callback) {
			var escapeReplace = function (char) {
				switch (char) {
			    	case '<': return '&lt;';
			    	case '>': return '&gt;';
			    	case '&': return '&amp;';
			    	case '"': return '&quot;';
			    	default: return char;
				}
			};
			var data = data.replace(/[&<>"]/g, escapeReplace);
			callback(undefined, data);
	},
	compress: function(data, callback) {
        callback(undefined, data
        		.replace(/  */g, ' ')
				.replace(/\n /g, '\n')
				.replace(/\n\n*/g, '\n')
        		);
    }
};

/**
 * Parses template file to tokenized stream.
 * 
 * @param origin Absolute path to file
 * @param filters Custom compile-phase filters
 * @param callback Function that takes two arguments: 
 * 				   Error and tokenized stream. 
 */
function parse(origin, customOptions, callback) {
	loader(origin, function(err, source) {
	parseText(source, origin, customOptions, function(err, stream) {
		sys.puts(sys.inspect(stream));
			callback(err, stream);
			
	});	});
}
exports.parse = parse;

/**
 * Parses template text to tokenized stream.
 * 
 * @param source Template code
 * @param origin Absolute path to root directory
 * @param filters Custom compile-phase filters
 * @param callback Function that takes two arguments: 
 * 				   Error and tokenized stream.
 */<commit_msg>Remove debug message<commit_after>var sys = require("sys");
var fs = require("fs");
var path = require("path");
var Buffer = require('buffer').Buffer;

var defaultFilters = {
	escape: function(data, callback) {
			var escapeReplace = function (char) {
				switch (char) {
			    	case '<': return '&lt;';
			    	case '>': return '&gt;';
			    	case '&': return '&amp;';
			    	case '"': return '&quot;';
			    	default: return char;
				}
			};
			var data = data.replace(/[&<>"]/g, escapeReplace);
			callback(undefined, data);
	},
	compress: function(data, callback) {
        callback(undefined, data
        		.replace(/  */g, ' ')
				.replace(/\n /g, '\n')
				.replace(/\n\n*/g, '\n')
        		);
    }
};

/**
 * Parses template file to tokenized stream.
 * 
 * @param origin Absolute path to file
 * @param filters Custom compile-phase filters
 * @param callback Function that takes two arguments: 
 * 				   Error and tokenized stream. 
 */
function parse(origin, customOptions, callback) {
	loader(origin, function(err, source) {
	parseText(source, origin, customOptions, function(err, stream) {
			callback(err, stream);
	});	});
}
exports.parse = parse;

/**
 * Parses template text to tokenized stream.
 * 
 * @param source Template code
 * @param origin Absolute path to root directory
 * @param filters Custom compile-phase filters
 * @param callback Function that takes two arguments: 
 * 				   Error and tokenized stream.
 */
{'fraction_non_alphanumeric': 0.15061475409836064, 'fraction_numerical': 0.0, 'mean_word_length': 3.0455801104972378, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5337852', 'n_tokens_mistral': 986, 'n_tokens_neox': 883, 'n_words': 269}