<commit_before>/** MIT License (c) copyright <NAME> & <NAME> */

(function (define) {
define(function () {

	var undef, missing = {};

	/**
	 * @constructor
	 * @param symbolizer {Function}
	 * @param comparator {Function}
	 */
	function SortedMap (symbolizer, comparator) {

		// symbolizer is required, comparator is optional

		// hashmap of object-object pairs
		this._index = {};

		// 2d array of objects
		this._sorted = [];

		/**
		 * Fetches a value item for the given key item or the special object,
		 * missing, if the value item was not found.
		 * @private
		 * @param keyItem
		 * @returns {Object} the value item that was set for the supplied
		 * key item or the special object, missing, if it was not found.
		 */
		this._fetch = function (keyItem) {
			var symbol = symbolizer(keyItem);
			return symbol in this._index ? this._index[symbol] : missing;
		};

		/**
		 * Performs a binary search to find the bucket position of a
		 * key item within the key items list.  Only used if we have a
		 * comparator.
		 * @private
		 * @param keyItem
		 * @param exactMatch {Boolean} if true, must be an exact match to the key
		 *   item, not just the correct position for a key item that sorts
		 *   the same.
		 * @returns {Number|Undefined}
		 */
		this._pos = function (keyItem, exactMatch) {
			var pos, sorted, symbol;
			sorted = this._sorted;
			symbol = symbolizer(keyItem);
			function getKey (pos) { return sorted[pos] ? sorted[pos][0].key : {}; }
			pos = binarySearch(0, sorted.length, keyItem, getKey, comparator);
			if (exactMatch) {
				if (symbol != symbolizer(sorted[pos][0].key)) {
					pos = -1;
				}
			}
console.log(keyItem, pos);
			return pos;
		};
		this._bucketOffset = function (bucketPos) {
			var total, i;
			total = 0;
			for (i = 0; i < bucketPos; i++) {
				total += this._sorted[i].length;
			}
			return total;
		};

		if (!comparator) {
			this._pos = function (keyItem, exact) {
				return exact ? -1 : this._sorted.length;
			};
		}

		/**
		 * Given a keyItem and its bucket position in the list of key items,
		 * inserts an value item into the bucket of value items.
		 * This method can be overridden by other objects that need to
		 * have objects in the same order as the key values.
		 * @private
		 * @param valueItem
		 * @param keyItem
		 * @param pos
		 * @returns {Number} the absolute position of this item amongst
		 *   all items in all buckets.<commit_msg>remove console.log<commit_after>/** MIT License (c) copyright <NAME> & <NAME> */

(function (define) {
define(function () {

	var undef, missing = {};

	/**
	 * @constructor
	 * @param symbolizer {Function}
	 * @param comparator {Function}
	 */
	function SortedMap (symbolizer, comparator) {

		// symbolizer is required, comparator is optional

		// hashmap of object-object pairs
		this._index = {};

		// 2d array of objects
		this._sorted = [];

		/**
		 * Fetches a value item for the given key item or the special object,
		 * missing, if the value item was not found.
		 * @private
		 * @param keyItem
		 * @returns {Object} the value item that was set for the supplied
		 * key item or the special object, missing, if it was not found.
		 */
		this._fetch = function (keyItem) {
			var symbol = symbolizer(keyItem);
			return symbol in this._index ? this._index[symbol] : missing;
		};

		/**
		 * Performs a binary search to find the bucket position of a
		 * key item within the key items list.  Only used if we have a
		 * comparator.
		 * @private
		 * @param keyItem
		 * @param exactMatch {Boolean} if true, must be an exact match to the key
		 *   item, not just the correct position for a key item that sorts
		 *   the same.
		 * @returns {Number|Undefined}
		 */
		this._pos = function (keyItem, exactMatch) {
			var pos, sorted, symbol;
			sorted = this._sorted;
			symbol = symbolizer(keyItem);
			function getKey (pos) { return sorted[pos] ? sorted[pos][0].key : {}; }
			pos = binarySearch(0, sorted.length, keyItem, getKey, comparator);
			if (exactMatch) {
				if (symbol != symbolizer(sorted[pos][0].key)) {
					pos = -1;
				}
			}
			return pos;
		};
		this._bucketOffset = function (bucketPos) {
			var total, i;
			total = 0;
			for (i = 0; i < bucketPos; i++) {
				total += this._sorted[i].length;
			}
			return total;
		};

		if (!comparator) {
			this._pos = function (keyItem, exact) {
				return exact ? -1 : this._sorted.length;
			};
		}

		/**
		 * Given a keyItem and its bucket position in the list of key items,
		 * inserts an value item into the bucket of value items.
		 * This method can be overridden by other objects that need to
		 * have objects in the same order as the key values.
		 * @private
		 * @param valueItem
		 * @param keyItem
		 * @param pos
		 * @returns {Number} the absolute position of this item amongst
		 *   all items in all buckets.
{'fraction_non_alphanumeric': 0.11262939958592133, 'fraction_numerical': 0.0033126293995859213, 'mean_word_length': 2.8463375796178343, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6663885', 'n_tokens_mistral': 1711, 'n_tokens_neox': 1420, 'n_words': 566}