<commit_before>//TODO fix difference between wsdl parsed by json-q and by old parser
//TODO make it works with browsers (IE9+)
//TODO  JSON.parse(JSON.stringify(ret)) is not good for Dates
//TODO shouldi add pseudo-classes like :empty :only-child :first-child :last-child :nth-child(n) :nth-last-child(n) :not(selector) ?
//TODO config or plugin system ( for filters and pseudos)
//TODO? should i add [x>25] and custom filter function?

const { parse } = require('./parse');
const { parse_filter } = require('./parse_filter');
const { operator } = require('./filter_operators');

const _dedup = x => {return (x instanceof Array ? x.filter((v, i, a) => a.indexOf(v) === i) : x)} //dedup array

const _deep_filter = (obj, before, after, parent, parent_key) => {
	let ret = obj;
	if (typeof obj == 'object') {
		if (before) ret = before(ret, parent, parent_key);
		for(let i in ret) {
			ret[i] = _deep_filter(obj[i], before, after, ret, i);
		}
		if (after) ret = after(ret, parent, parent_key);
	}
	return ret;
}

const get = (obj, path) => {
	return _get(obj, parse(path));
}

const _get = (obj, flow) => {
	flow = Object.assign([], flow);
	let ret = [];
	if (obj instanceof Array) {<commit_msg>add TODOs<commit_after>//FIX does it really flatten arrays? test [[[[[{a:1}]]]]]
//FIX JSON.parse(JSON.stringify(ret)) is not good for Dates
//FIX " * " means nothing while ".*" means "get next level of hierarchy" 
//    i.e ".*.b" of [{b:1}, c:{b:2}] should return [2]
//        " * b" of [{b:1}, c:{b:2}] should return [1, 2]
//FIX fix difference between wsdl parsed by json-q and by old parser

//TODO add pseudo-classes like :empty :only-child :first-child :last-child :nth-child(n) :nth-last-child(n) :not(selector) ?
//TODO config to add/change filters and pseudos
//TODO make it works with browsers (IE9+)
//TODO? should i add [x>25] and custom filter function? - looks like make it via pseudos is a good idea

const { parse } = require('./parse');
const { parse_filter } = require('./parse_filter');
const { operator } = require('./filter_operators');

const _dedup = x => {return (x instanceof Array ? x.filter((v, i, a) => a.indexOf(v) === i) : x)} //dedup array

const _deep_filter = (obj, before, after, parent, parent_key) => {
	let ret = obj;
	if (typeof obj == 'object') {
		if (before) ret = before(ret, parent, parent_key);
		for(let i in ret) {
			ret[i] = _deep_filter(obj[i], before, after, ret, i);
		}
		if (after) ret = after(ret, parent, parent_key);
	}
	return ret;
}

const get = (obj, path) => {
	return _get(obj, parse(path));
}

const _get = (obj, flow) => {
	flow = Object.assign([], flow);
	let ret = [];
	if (obj instanceof Array) {
{'fraction_non_alphanumeric': 0.1741256111320045, 'fraction_numerical': 0.0052651372696502444, 'mean_word_length': 3.854014598540146, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1099101', 'n_tokens_mistral': 948, 'n_tokens_neox': 925, 'n_words': 318}