<commit_before>module.exports = function diffResource(src, dst, opts) {
	if (!opts) opts = {};
	var equal = opts.equal || defaultEqual();
	var key = opts.key || 'id';

	var transforms = {put: [], post: [], del: []};

	var hash = {};
	var pk;

	src.forEach(function(srcItem) {
		var pk = srcItem[key];
		if (!hash[pk]) hash[pk] = [];
		hash[pk].push(srcItem);
	});

	dst.forEach(function(dstItem) {
		var pk = dstItem[key];
		var srcItems = hash[pk];
		if (srcItems && srcItems.length) {
			var srcItem = srcItems.shift();
			if (!equal(srcItem, dstItem)) {
				transforms.put.push(dstItem);
			}
			if (srcItems.length == 0) delete hash[pk];
		} else {
			transforms.post.push(dstItem);
		}
	});

	for (pk in hash) transforms.del = transforms.del.concat(hash[pk]);

	return transforms;
}

function defaultEqual() {
	var deepEqual = require('deep-equal');
	return function(a, b) {
		return deepEqual(a, b, {strict: true});
	}
}
<commit_msg>Better support duplicates, try to find a match<commit_after>module.exports = function diffResource(src, dst, opts) {
	if (!opts) opts = {};
	var equal = opts.equal || defaultEqual();
	var key = opts.key || 'id';

	var transforms = {put: [], post: [], del: []};

	var hash = {};
	var pk;

	src.forEach(function(srcItem) {
		var pk = srcItem[key];
		if (!hash[pk]) hash[pk] = [];
		hash[pk].push(srcItem);
	});

	dst.forEach(function(dstItem) {
		var pk = dstItem[key];
		// keep only different items
		var found = false;
		var srcItems = (hash[pk] || []).filter(function(srcItem) {
			var same = equal(srcItem, dstItem);
			if (!found && same) found = true;
			return !same;
		});
		hash[pk] = srcItems;
		if (!found) {
			if (srcItems.length) {
				// update first item
				var srcItem = srcItems.shift();
				if (!equal(srcItem, dstItem)) {
					transforms.put.push(dstItem);
				}
			} else {
				// new item
				transforms.post.push(dstItem);
			}
		}
		if (srcItems.length == 0) delete hash[pk];
	});

	for (pk in hash) transforms.del = transforms.del.concat(hash[pk]);

	return transforms;
}

function defaultEqual() {
	var deepEqual = require('deep-equal');
	return function(a, b) {
		return deepEqual(a, b, {strict: true});
	}
}

{'fraction_non_alphanumeric': 0.17885057471264368, 'fraction_numerical': 0.0009195402298850574, 'mean_word_length': 3.5714285714285716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '150575', 'n_tokens_mistral': 869, 'n_tokens_neox': 774, 'n_words': 190}