<commit_before>"use strict";

function jsonop(oa, ob, oop) {
	function fn (oa, ob, oop) {
		const el = { _: oa }, stack = [ { a: el, b: { _: ob }, op: { _: oop } } ];

		const opfn = {
			keep: (a) => a,
			inc: (a, b, params) => (params && params[0]) ? (a + b) % params[0] : a + b,
			mul: (a, b, params) => (params && params[0]) ? (a * b) % params[0] : a * b,
			min: (a, b) => Math.min(a, b),
			max: (a, b) => Math.max(a, b),
			mavg: (a, b, params) => (a * (params[0] - 1) + b) / params[0],
			union: (a, b, p) => {
				console.log("union", a,b,p);
				var arr = Array.isArray(b) ? a.concat(b) : a, result = [], map = {},
					params = p || [];

				arr.forEach(function(e) {
					if (!map[e] && params.indexOf(e) < 0) result.push(e);
					map[e] = true;
				});

				return result;
			},
			inter: (a, b, params) => {
				const map = {};

				for (const i of b) { if (i in a) { map[i] = true; } }
				if (params) for (const i in params) { map[i] = true; }

				return Object.keys(map);
			},
			splice: (a, b, params) => {
				if (params.length >= 2) {
					a.splice(
						params[0] === null ? Infinity : params[0], params[1], ...b
					);
				}
				if (params.length >= 4) {
					return a.slice(
						params[2] === null ? Infinity : params[2],
						params[3] === null ? Infinity : params[3]
					);
				} else { return a; }
			},
			replace: (a, b) => b,
			append: (a, b, params) => a + (params[0] || "") + b,
			band: (a, b) => a & b,
			bor: (a, b) => a | b,
			bxor: (a, b) => a ^ b,
			and: (a, b) => a && b,
			or: (a, b) => a || b,
			not: (a) => !a
			// delete and merge are handled separately below
		};

		function deleteOps(obj) {
			const stack = [ obj ];

			do {
				const o = stack.pop();

				delete stack.__op__;
				for (const i in o) {
					if (typeof o[i] === "object" && o[i] !== null) {
						stack.push(o[i]);
					}
				}
			} while (stack.length);
			return obj;
		}


		function isOp(op) {
			return typeof op === "string" || Array.isArray(op);
		}

		function opval(a, b, i, op, stack) {

			console.log(a, b, i, op, stack);

			if (op === "delete" || op && op[0] === "delete") {
				delete a[i];
			} else if (op && isOp(op) && op !== "merge" && op[0] !== "merge") {
				if (Array.isArray(op)) {
					a[i] = deleteOps(opfn[op.shift()](a[i], b[i], op));
				} else {
					a[i] = deleteOps(opfn[op](a[i], b[i]));
				}
			} else if (
				Array.isArray(b[i]) && Array.isArray(a[i]) &&
				(op === "merge" || op && op[0] === "merge")
			) {
				stack.push({ a: a[i], b: b[i] });
			} else if (Array.isArray(b) && b[i] === null) {
				// While merging arrays, skip nulls
			} else if (
				typeof b[i] === "object" && typeof a[i] === "object" &&
				b[i] !== null && a[i] !== null &&
				!Array.isArray(a[i]) && !Array.isArray(b[i])
			) {
				stack.push({ a: a[i], b: b[i], op });
			} else {
				a[i] = b[i];
			}
		}

		let count = 0;
		do {
			const frame = stack.pop(),
				a = frame.a, b = frame.b;<commit_msg>Prevent the original op given.<commit_after>"use strict";

function jsonop(oa, ob, oop) {
	function fn (oa, ob, oop) {
		const el = { _: oa }, stack = [ { a: el, b: { _: ob }, op: { _: oop } } ];

		const opfn = {
			keep: (a) => a,
			inc: (a, b, params) => (params && params[0]) ? (a + b) % params[0] : a + b,
			mul: (a, b, params) => (params && params[0]) ? (a * b) % params[0] : a * b,
			min: (a, b) => Math.min(a, b),
			max: (a, b) => Math.max(a, b),
			mavg: (a, b, params) => (a * (params[0] - 1) + b) / params[0],
			union: (a, b, p) => {
				console.log("union", a,b,p);
				var arr = Array.isArray(b) ? a.concat(b) : a, result = [], map = {},
					params = p || [];

				arr.forEach(function(e) {
					if (!map[e] && params.indexOf(e) < 0) result.push(e);
					map[e] = true;
				});

				return result;
			},
			inter: (a, b, params) => {
				const map = {};

				for (const i of b) { if (i in a) { map[i] = true; } }
				if (params) for (const i in params) { map[i] = true; }

				return Object.keys(map);
			},
			splice: (a, b, params) => {
				if (params.length >= 2) {
					a.splice(
						params[0] === null ? Infinity : params[0], params[1], ...b
					);
				}
				if (params.length >= 4) {
					return a.slice(
						params[2] === null ? Infinity : params[2],
						params[3] === null ? Infinity : params[3]
					);
				} else { return a; }
			},
			replace: (a, b) => b,
			append: (a, b, params) => a + (params[0] || "") + b,
			band: (a, b) => a & b,
			bor: (a, b) => a | b,
			bxor: (a, b) => a ^ b,
			and: (a, b) => a && b,
			or: (a, b) => a || b,
			not: (a) => !a
			// delete and merge are handled separately below
		};

		function deleteOps(obj) {
			const stack = [ obj ];

			do {
				const o = stack.pop();

				delete stack.__op__;
				for (const i in o) {
					if (typeof o[i] === "object" && o[i] !== null) {
						stack.push(o[i]);
					}
				}
			} while (stack.length);
			return obj;
		}


		function isOp(op) {
			return typeof op === "string" || Array.isArray(op);
		}

		function opval(a, b, i, op, stack) {

			console.log(a, b, i, op, stack);

			if (op === "delete" || op && op[0] === "delete") {
				delete a[i];
			} else if (op && isOp(op) && op !== "merge" && op[0] !== "merge") {
				if (Array.isArray(op)) {
					a[i] = deleteOps(opfn[op[0]](a[i], b[i], op));
				} else {
					a[i] = deleteOps(opfn[op](a[i], b[i]));
				}
			} else if (
				Array.isArray(b[i]) && Array.isArray(a[i]) &&
				(op === "merge" || op && op[0] === "merge")
			) {
				stack.push({ a: a[i], b: b[i] });
			} else if (Array.isArray(b) && b[i] === null) {
				// While merging arrays, skip nulls
			} else if (
				typeof b[i] === "object" && typeof a[i] === "object" &&
				b[i] !== null && a[i] !== null &&
				!Array.isArray(a[i]) && !Array.isArray(b[i])
			) {
				stack.push({ a: a[i], b: b[i], op });
			} else {
				a[i] = b[i];
			}
		}

		let count = 0;
		do {
			const frame = stack.pop(),
				a = frame.a, b = frame.b;
{'fraction_non_alphanumeric': 0.2418809201623816, 'fraction_numerical': 0.0076116373477672535, 'mean_word_length': 2.250687190764156, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6459194', 'n_tokens_mistral': 2960, 'n_tokens_neox': 2450, 'n_words': 653}