<commit_before>'use strict';

function merge(query, next) {
	if (Array.isArray(query)) {
		return query.map(q => merge(q, next))
	}
	if (query.aggregator) {
		query.body.forEach(q => {
			merge(q, next);
		})
		return query;
	} else if (query.functions) {
		query.functions = query.functions.concat(next);
		return query;
	} else {
		return [query].concat(next)
	}
}

function analyseFunctionCall (func, body) {
	if (typeof body === 'string' && /^('|").*\1/.test(body)) {
		body = body.substr(1, body.length -2);
	}

	return func.charAt(0) === '@' ? {aggregator: func, body} : {func, body};
}

class Deconstructor {
	constructor (aggregator) {
		this.aggregator = aggregator;
		this.openSingleQuotes = 0;
		this.openDoubleQuotes = 0;
	}

	deconstructAggregator (str) {

		this.str = str;

		if (str.charAt(0) === '@') {
			return this.deconstruct(str)
		}

		const queries = [];
		while(this.str) {
			const str = this.str;
			this.str = null;
			queries.push(str.charAt(0) === '@' ? this.deconstruct(str) : this.deconstructQuery(str));
		}
		return queries;
	}

	deconstructQuery (str) {

		if (str.charAt(0) === '@') {
			return this.deconstructAggregator(str);
		}
		const firstCut = str.split('->');
		const eventName = firstCut.shift();
		str = firstCut.join('->');
		const functions = this.deconstruct(str);
		return {
			event: eventName,
			functions: functions
		}
	}

	deconstruct (str) {

		if (!str) {
			return str;
		}

		if (this.aggregator && str.indexOf(',') === 0) {
			this.str = str.substr(1);
			return;
		}

		str = str.replace(/^->/, '');

		let openParantheses = 0;
		let char;
		let func;
		let body;

		for (let i = 0, il = str.length; i < il; i++) {
			char = str.charAt(i);
			switch (char) {
				case '(':
					!this.openDoubleQuotes && !this.openSingleQuotes && openParantheses++;
					break;
				case ')':
					!this.openDoubleQuotes && !this.openSingleQuotes && openParantheses--;
					break;
				case '"':
					!this.openSingleQuotes && (this.openDoubleQuotes = Math.abs(this.openDoubleQuotes - 1));
					break;
				case '\'':
					!this.openDoubleQuotes && (this.openSingleQuotes = Math.abs(this.openSingleQuotes - 1));
					break;
				case 'default':
					break;
			}


			if (openParantheses && !func) {
				func = str.substr(0, i);
			} else if (func && !openParantheses && !body) {
				body = str.substr(func.length + 1, i - (func.length + 1))
				if (func.charAt(0) === '@') {
					body = new Deconstructor(func.substr(1)).deconstructAggregator(body);
				}
				let res = analyseFunctionCall(func, body);
				const next = this.deconstruct(str.substr(i + 1), this.openSingleQuotes, this.openDoubleQuotes)
				return next ? merge(res, next) : res.aggregator ? res : [res]
			}
		}
		return str;
	}
}
module.exports = function deconstruct (str) {
	return new Deconstructor().deconstructQuery(str);
}
<commit_msg>handle printing<commit_after>'use strict';

function merge(query, next) {
	if (Array.isArray(query)) {
		return query.map(q => merge(q, next))
	}
	if (query.aggregator) {
		query.body.forEach(q => {
			merge(q, next);
		})
		return query;
	} else if (query.functions) {
		query.functions = query.functions.concat(next);
		return query;
	} else {
		return [query].concat(next)
	}
}

function analyseFunctionCall (func, body) {
	if (typeof body === 'string' && /^('|").*\1/.test(body)) {
		body = body.substr(1, body.length -2);
	}

	return func.charAt(0) === '@' ? {aggregator: func, body} : {func, body};
}

class Deconstructor {
	constructor (aggregator) {
		this.aggregator = aggregator;
		this.openSingleQuotes = 0;
		this.openDoubleQuotes = 0;
	}

	deconstructAggregator (str) {

		this.str = str;

		if (str.charAt(0) === '@') {
			const query = this.deconstruct(str);
			query.print = this.topLevel && this.print
			return query;
		}

		const queries = [];
		while(this.str) {
			const str = this.str;
			this.str = null;
			queries.push(str.charAt(0) === '@' ? this.deconstruct(str) : this.deconstructQuery(str));
		}
		return queries;
	}

	deconstructQuery (str) {

		if (str.charAt(0) === '@') {
			return this.deconstructAggregator(str);
		}
		const firstCut = str.split('->');
		const eventName = firstCut.shift();
		str = firstCut.join('->');
		const functions = this.deconstruct(str);
		return {
			event: eventName,
			functions: functions,
			print: this.topLevel && this.print
		}
	}

	deconstruct (str) {

		if (!str) {
			return str;
		}

		if (this.aggregator && str.indexOf(',') === 0) {
			this.str = str.substr(1);
			return;
		}

		str = str.replace(/^->/, '');

		let openParantheses = 0;
		let char;
		let func;
		let body;

		for (let i = 0, il = str.length; i < il; i++) {
			char = str.charAt(i);
			switch (char) {
				case '(':
					!this.openDoubleQuotes && !this.openSingleQuotes && openParantheses++;
					break;
				case ')':
					!this.openDoubleQuotes && !this.openSingleQuotes && openParantheses--;
					break;
				case '"':
					!this.openSingleQuotes && (this.openDoubleQuotes = Math.abs(this.openDoubleQuotes - 1));
					break;
				case '\'':
					!this.openDoubleQuotes && (this.openSingleQuotes = Math.abs(this.openSingleQuotes - 1));
					break;
				case 'default':
					break;
			}


			if (openParantheses && !func) {
				func = str.substr(0, i);
			} else if (func && !openParantheses && !body) {
				body = str.substr(func.length + 1, i - (func.length + 1))
				if (func.charAt(0) === '@') {
					body = new Deconstructor(func.substr(1)).deconstructAggregator(body);
				}
				if (func === 'print') {
					this.print = body;
				}
				let res = analyseFunctionCall(func, body);
				const next = this.deconstruct(str.substr(i + 1), this.openSingleQuotes, this.openDoubleQuotes)
				return next ? merge(res, next) : res.aggregator ? res : [res]
			}
		}
		return str;
	}
}
module.exports = function deconstruct (str) {
	const deconstructor = new Deconstructor();
	deconstructor.topLevel = true;
	return deconstructor.deconstructQuery(str);
}

{'fraction_non_alphanumeric': 0.1725358045492839, 'fraction_numerical': 0.007076663858466723, 'mean_word_length': 3.1920903954802258, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1012128', 'n_tokens_mistral': 2494, 'n_tokens_neox': 2115, 'n_words': 442}