<commit_before>"use strict";
var RuleSet = require('./rules/rule_set');
var FixKeys = require('./rules/others/fix_keys');


var Repacker = function (opt_rules, opt_othersRule) {
	this.rules = opt_rules || new RuleSet();
	this.othersRule = opt_othersRule;

	this.dstKeyRuleMap = null;
	this.ruleCoveredSrcKeys = null;

	this.collectingFunc = null;
	this.transformFunc = null;

	this.keyExprMap = null;
};

Repacker.prototype.getKeyExprMap = function () {
	return this.keyExprMap;
};

Repacker.prototype.srcKeysToFields = function (srcKeys, prefix, outPrefix) {
	var keyExprMap = this.getKeyExprMap();
	var result;
	if (keyExprMap == null && !prefix && !outPrefix) {
		result = srcKeys;
	}
	else {
		result = [];
		for (var i = 0; i < srcKeys.length; i++) {
			var srcKey = srcKeys[i];
			var expr = null;
			if (keyExprMap != null) {
				var exprFunc = keyExprMap[srcKey];
				if (exprFunc) {
					expr = exprFunc(prefix);
				}
			}
			var str = expr || this.prefixSrcKey(srcKey, prefix);
			if (expr || outPrefix) {
				var alias = this.prefixField(srcKey, outPrefix);
				str = [str, alias].join(' as ');
			}
			result.push(str);
		}
	}
	return result.join(', ');
};

Repacker.prototype.getFields = function (dstKeys, prefix, outPrefix) {
	return this.srcKeysToFields(this.getSrcKeys(dstKeys), prefix, outPrefix);
};

Repacker.prototype.getCollectingFunc = function () {
	if (this.collectingFunc == null) {
		var self = this;
		this.collectingFunc = function (row, data) {
			data.push(self.repack(row));
		};
	}
	return this.collectingFunc;
};

Repacker.prototype.getTransformFunc = function () {
	if (this.transformFunc == null) {
		var self = this;
		this.transformFunc = function (row) {
			return self.repack(row);
		};
	}
	return this.transformFunc;
};

Repacker.prototype.addRule = function (rule) {
	this.rules.add(rule);
};

Repacker.prototype.getRuleForDstKey = function (dstKey) {
	if (this.dstKeyRuleMap == null) {
		this.dstKeyRuleMap = {};
		this.rules.addToDstKeyRuleMap(this.dstKeyRuleMap);
	}
	return this.dstKeyRuleMap[dstKey];
};

Repacker.prototype.isSrcKeyCoveredByRule = function (srcKey) {
	if (this.ruleCoveredSrcKeys == null) {
		this.ruleCoveredSrcKeys = {};
		this.rules.addCoversSrcKeysToSet(this.ruleCoveredSrcKeys);
	}
	return this.ruleCoveredSrcKeys[srcKey];
};

Repacker.prototype.getSrcKeys = function (dstKeys) {
	var result = new Repacker.KeySet(), otherDstKeys = [];
	for (var i = 0; i < dstKeys.length; i++) {
		var k = dstKeys[i];
		var rule = this.getRuleForDstKey(k);
		if (rule != null) {
			rule.addDependsOnSrcKeys(result);
		}
		else if (this.othersRule != null) {
			otherDstKeys.push(k);
		}
		else {
			throw new Error('Unknown dst key ' + k);
		}
	}
	if (this.othersRule != null) {
		this.othersRule.addDependsOnSrcKeys(result, otherDstKeys);
	}
	return result.arr;
};

Repacker.prototype.repack = function (src, opt_dstKeys) {
	var dst = null;
	if (src != null) {
		dst = {};
		var i, k, rule, otherDstKeys, skipFunc;
		if (opt_dstKeys) {
			otherDstKeys = [];
			for (i = 0; i < opt_dstKeys.length; i++) {
				k = opt_dstKeys[i];
				rule = this.getRuleForDstKey(k);
				if (rule != null) {
					rule.repack(src, dst);
				}
				else if (this.othersRule != null) {
					otherDstKeys.push(k);
				}
				else {
					throw new Error('Unknown dst key ' + k);
				}
			}
		}
		else {
			this.rules.repack(src, dst);
			var self = this;
			skipFunc = function (dstKey) {
				return self.isSrcKeyCoveredByRule(dstKey);
			};
		}
		if (this.othersRule != null) {
			this.othersRule.repack(src, dst, otherDstKeys, skipFunc);
		}
	}
	return dst;
};

Repacker.prototype.reverse = function (dst, opt_dstKeys) {
	var src = {};
	var i, k, rule, otherDstKeys, skipFunc;
	if (opt_dstKeys) {
		otherDstKeys = [];
		for (i = 0; i < opt_dstKeys.length; i++) {
			k = opt_dstKeys[i];
			rule = this.getRuleForDstKey(k);
			if (rule != null) {
				rule.reverse(dst, src);
			}
			else if (this.othersRule != null) {
				otherDstKeys.push(k);
			}
			else {
				throw new Error('Unknown dst key ' + k);
			}
		}
	}
	else {
		this.rules.reverse(src, dst);<commit_msg>fixed broken reverse<commit_after>"use strict";
var RuleSet = require('./rules/rule_set');
var FixKeys = require('./rules/others/fix_keys');


var Repacker = function (opt_rules, opt_othersRule) {
	this.rules = opt_rules || new RuleSet();
	this.othersRule = opt_othersRule;

	this.dstKeyRuleMap = null;
	this.ruleCoveredSrcKeys = null;

	this.collectingFunc = null;
	this.transformFunc = null;

	this.keyExprMap = null;
};

Repacker.prototype.getKeyExprMap = function () {
	return this.keyExprMap;
};

Repacker.prototype.srcKeysToFields = function (srcKeys, prefix, outPrefix) {
	var keyExprMap = this.getKeyExprMap();
	var result;
	if (keyExprMap == null && !prefix && !outPrefix) {
		result = srcKeys;
	}
	else {
		result = [];
		for (var i = 0; i < srcKeys.length; i++) {
			var srcKey = srcKeys[i];
			var expr = null;
			if (keyExprMap != null) {
				var exprFunc = keyExprMap[srcKey];
				if (exprFunc) {
					expr = exprFunc(prefix);
				}
			}
			var str = expr || this.prefixSrcKey(srcKey, prefix);
			if (expr || outPrefix) {
				var alias = this.prefixField(srcKey, outPrefix);
				str = [str, alias].join(' as ');
			}
			result.push(str);
		}
	}
	return result.join(', ');
};

Repacker.prototype.getFields = function (dstKeys, prefix, outPrefix) {
	return this.srcKeysToFields(this.getSrcKeys(dstKeys), prefix, outPrefix);
};

Repacker.prototype.getCollectingFunc = function () {
	if (this.collectingFunc == null) {
		var self = this;
		this.collectingFunc = function (row, data) {
			data.push(self.repack(row));
		};
	}
	return this.collectingFunc;
};

Repacker.prototype.getTransformFunc = function () {
	if (this.transformFunc == null) {
		var self = this;
		this.transformFunc = function (row) {
			return self.repack(row);
		};
	}
	return this.transformFunc;
};

Repacker.prototype.addRule = function (rule) {
	this.rules.add(rule);
};

Repacker.prototype.getRuleForDstKey = function (dstKey) {
	if (this.dstKeyRuleMap == null) {
		this.dstKeyRuleMap = {};
		this.rules.addToDstKeyRuleMap(this.dstKeyRuleMap);
	}
	return this.dstKeyRuleMap[dstKey];
};

Repacker.prototype.isSrcKeyCoveredByRule = function (srcKey) {
	if (this.ruleCoveredSrcKeys == null) {
		this.ruleCoveredSrcKeys = {};
		this.rules.addCoversSrcKeysToSet(this.ruleCoveredSrcKeys);
	}
	return this.ruleCoveredSrcKeys[srcKey];
};

Repacker.prototype.getSrcKeys = function (dstKeys) {
	var result = new Repacker.KeySet(), otherDstKeys = [];
	for (var i = 0; i < dstKeys.length; i++) {
		var k = dstKeys[i];
		var rule = this.getRuleForDstKey(k);
		if (rule != null) {
			rule.addDependsOnSrcKeys(result);
		}
		else if (this.othersRule != null) {
			otherDstKeys.push(k);
		}
		else {
			throw new Error('Unknown dst key ' + k);
		}
	}
	if (this.othersRule != null) {
		this.othersRule.addDependsOnSrcKeys(result, otherDstKeys);
	}
	return result.arr;
};

Repacker.prototype.repack = function (src, opt_dstKeys) {
	var dst = null;
	if (src != null) {
		dst = {};
		var i, k, rule, otherDstKeys, skipFunc;
		if (opt_dstKeys) {
			otherDstKeys = [];
			for (i = 0; i < opt_dstKeys.length; i++) {
				k = opt_dstKeys[i];
				rule = this.getRuleForDstKey(k);
				if (rule != null) {
					rule.repack(src, dst);
				}
				else if (this.othersRule != null) {
					otherDstKeys.push(k);
				}
				else {
					throw new Error('Unknown dst key ' + k);
				}
			}
		}
		else {
			this.rules.repack(src, dst);
			var self = this;
			skipFunc = function (dstKey) {
				return self.isSrcKeyCoveredByRule(dstKey);
			};
		}
		if (this.othersRule != null) {
			this.othersRule.repack(src, dst, otherDstKeys, skipFunc);
		}
	}
	return dst;
};

Repacker.prototype.reverse = function (dst, opt_dstKeys) {
	var src = {};
	var i, k, rule, otherDstKeys, skipFunc;
	if (opt_dstKeys) {
		otherDstKeys = [];
		for (i = 0; i < opt_dstKeys.length; i++) {
			k = opt_dstKeys[i];
			rule = this.getRuleForDstKey(k);
			if (rule != null) {
				rule.reverse(dst, src);
			}
			else if (this.othersRule != null) {
				otherDstKeys.push(k);
			}
			else {
				throw new Error('Unknown dst key ' + k);
			}
		}
	}
	else {
		this.rules.reverse(dst, src);
{'fraction_non_alphanumeric': 0.15046945494451897, 'fraction_numerical': 0.0009754907938056335, 'mean_word_length': 3.5541365907828983, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6489575', 'n_tokens_mistral': 3365, 'n_tokens_neox': 3018, 'n_words': 627}