<commit_before>const
	extend = require("extend"),
	Processor = require("./"),
	logger = require("../logger"),
	jsexpr = require("jsexpr");

class KeyValParserProcessor extends Processor {
	constructor(id,type) {
		super(id,type);
	}

	configure(config,callback) {
		this.config = extend({},config);
		this.output = this.config.output? jsexpr.assign(this.config.output) : null;
		this.input = jsexpr.expr(this.config.input || "${originalMessage}");

		callback();
	}

	start(callback) {
		callback();
	}

	process(entry,callback) {
		let msg = this.input(entry);
		let tokens = [], tlen = 0;
		let res = {};

		let p1 = msg.split(" "), p1len = p1.length;
		for(let i=0;i<p1len;i++) {
			let p2 = p1[i].split("="), p2len = p2.length;
			for(let j=0;j<p2len;j++) {
				tokens[tlen++] = p2[j];
			}
		}

		let haskey = false, token = null;
		while(tokens.length) {
			if(!haskey) {
				haskey = true;
				token = tokens.shift();
				res[token] = "";
			}
			else {
				haskey = false;
				let val = tokens.shift();

				// Simple value without quotes
				if(!val.startsWith('"')) {
					res[token] = val;
					haskey = false;
				}
				// Simple Quoted value
				else if(val.startsWith('"') && val.endsWith('"')){
					res[token] = val.substring(1,val.length-1);
				}
				// Spaced quoted value
				else {
					res[token] += val;
					do {
						val = tokens.shift();
						res[token] += " " + val;
					}while(!val.endsWith('"'));<commit_msg>keyval parser bug<commit_after>const
	extend = require("extend"),
	Processor = require("./"),
	logger = require("../logger"),
	jsexpr = require("jsexpr");

class KeyValParserProcessor extends Processor {
	constructor(id,type) {
		super(id,type);
	}

	configure(config,callback) {
		this.config = extend({},config);
		this.output = this.config.output? jsexpr.assign(this.config.output) : null;
		this.input = jsexpr.expr(this.config.input || "${originalMessage}");

		callback();
	}

	start(callback) {
		callback();
	}

	process(entry,callback) {
		let msg = this.input(entry);
		let tokens = [], tlen = 0;
		let res = {};

		let p1 = msg.split(" "), p1len = p1.length;
		for(let i=0;i<p1len;i++) {
			let p2 = p1[i].split("="), p2len = p2.length;
			for(let j=0;j<p2len;j++) {
				tokens[tlen++] = p2[j];
			}
		}

		let haskey = false, token = null;
		while(tokens.length) {
			if(!haskey) {
				haskey = true;
				token = tokens.shift();
				res[token] = "";
			}
			else {
				haskey = false;
				let val = tokens.shift();

				// Simple value without quotes
				if(!val.startsWith('"')) {
					res[token] = val;
					haskey = false;
				}
				// Simple Quoted value
				else if(val.startsWith('"') && val.endsWith('"')){
					res[token] = val.substring(1,val.length-1);
				}
				// Spaced quoted value
				else {
					res[token] += val;
					do {
						val = tokens.shift();
						res[token] += " " + val;
					}while(!val.endsWith('"'));
					res[token] = res[token].substring(1,val.length-1);
{'fraction_non_alphanumeric': 0.19319727891156463, 'fraction_numerical': 0.010884353741496598, 'mean_word_length': 2.963611859838275, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7460810', 'n_tokens_mistral': 1297, 'n_tokens_neox': 1076, 'n_words': 205}