<commit_before>const lexed = require('lexed');
lexed.english();
const lexer = lexed.lexer;
const tagger = require('en-pos');
const parser = require('en-parse');

var langr = function(input){
	var lexResult = lexer(input);
	var tagResult = lexResult.map(sentence=>tagger(sentence.tokens,sentence.meta));
	var parseReulst = lexResult.map((sentence,i)=>parser(tagResult[i].tags,uncontract(sentence.tokens)));
	this.input = input;
	this.sentences = lexResult.map(sentence=>sentence.raw);
	this.output = lexResult.map((sentence,index)=>{
		return {
			raw:this.sentences[index],
			tokens:lexResult[index].tokens,
			tokenizationMeta:lexResult[index].meta,
			confidence:tagResult[index].confidence-10,
			tags:tagResult[index].tags,
			deps:parseReulst[index],
		};
	});
	return this;
};

langr.extend = function(detector){
	if(Array.isArray(detector)) detector.forEach((single)=>add(single));
	else add(detector);
	function add(detector){
		if(typeof detector === "object" && detector !== null && detector.id && detector.detector) langr.prototype[detector.id] = detector.detector;<commit_msg>add interception ability<commit_after>const lexed = require('lexed');
lexed.english();
const lexer = lexed.lexer;
const tagger = require('en-pos');
const parser = require('en-parse');

var interceptors = [];

var fin = function(input){
	// process interceptions
	interceptors.forEach(f=>input=f(input));
	// lex, tag then parse ..
	var lexResult = lexer(input);
	var tagResult = lexResult.map(sentence=>tagger(sentence.tokens,sentence.meta));
	var parseReulst = lexResult.map((sentence,i)=>parser(tagResult[i].tags,uncontract(sentence.tokens)));
	this.input = input;
	this.sentences = lexResult.map(sentence=>sentence.raw);
	this.output = lexResult.map((sentence,index)=>{
		return {
			raw:this.sentences[index],
			tokens:lexResult[index].tokens,
			tokenizationMeta:lexResult[index].meta,
			confidence:tagResult[index].confidence-10,
			tags:tagResult[index].tags,
			deps:parseReulst[index],
		};
	});
	return this;
};


fin.intercept = function(f){
	if(typeof f !== "function") console.warn("FIN: An interceptor must be a function");
	else interceptors.push(f);
};

fin.extend = function(detector){
	if(Array.isArray(detector)) detector.forEach((single)=>add(single));
	else add(detector);
	function add(detector){
		if(typeof detector === "object" && detector !== null && detector.id && detector.detector) langr.prototype[detector.id] = detector.detector;
{'fraction_non_alphanumeric': 0.1660546715626275, 'fraction_numerical': 0.0016319869441044472, 'mean_word_length': 6.169590643274854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '992686', 'n_tokens_mistral': 873, 'n_tokens_neox': 796, 'n_words': 138}