<commit_before>/*
	MIT License http://www.opensource.org/licenses/mit-license.php
*/

"use strict";

const MapObjectSerializer = require("./MapObjectSerializer");
const PlainObjectSerializer = require("./PlainObjectSerializer");
const SerializerMiddleware = require("./SerializerMiddleware");
const SetObjectSerializer = require("./SetObjectSerializer");

/** @typedef {new (...params: any[]) => any} Constructor */

/*

Format:

File -> Section*
Section -> ObjectSection | ReferenceSection | EscapeSection | OtherSection

ObjectSection -> ESCAPE (
	number:relativeOffset (number > 0) |
	string:request (string|null):export
) Section:value* ESCAPE ESCAPE_END_OBJECT
ReferenceSection -> ESCAPE number:relativeOffset (number < 0)
EscapeSection -> ESCAPE ESCAPE_ESCAPE_VALUE (escaped value ESCAPE)
EscapeSection -> ESCAPE ESCAPE_UNDEFINED (escaped value ESCAPE)
OtherSection -> any (except ESCAPE)

Why using null as escape value?
Multiple null values can merged by the BinaryMiddleware, which makes it very efficient
Technically any value can be used.

*/

/**
 * @typedef {Object} ObjectSerializerContext
 * @property {function(any): void} write
 */

/**
 * @typedef {Object} ObjectDeserializerContext
 * @property {function(): any} read
 */

/**
 * @typedef {Object} ObjectSerializer
 * @property {function(any, ObjectSerializerContext): void} serialize
 * @property {function(ObjectDeserializerContext): any} deserialize
 */

const ESCAPE = null;
const ESCAPE_ESCAPE_VALUE = null;
const ESCAPE_END_OBJECT = true;
const ESCAPE_UNDEFINED = false;

const CURRENT_VERSION = 1;

const plainObjectSerializer = new PlainObjectSerializer();
const mapObjectSerializer = new MapObjectSerializer();
const setObjectSerializer = new SetObjectSerializer();

const serializers = new Map();
const serializerInversed = new Map();

const loadedRequests = new Set();

serializers.set(Object, {
	request: "",
	name: null,
	serializer: plainObjectSerializer
});
serializers.set(Array, {
	request: "",
	name: null,
	serializer: plainObjectSerializer
});
serializers.set(Map, {
	request: "",
	name: 1,
	serializer: mapObjectSerializer
});
serializers.set(Set, {
	request: "",
	name: 2,
	serializer: setObjectSerializer
});
for (const { request, name, serializer } of serializers.values()) {
	serializerInversed.set(`${request}/${name}`, serializer);
}

class ObjectMiddleware extends SerializerMiddleware {
	/**
	 * @param {Constructor} Constructor the constructor
	 * @param {string} request the request which will be required when deserializing
	 * @param {string} name the name to make multiple serializer unique when sharing a request
	 * @param {ObjectSerializer} serializer the serializer
	 * @returns {void}
	 */
	static register(Constructor, request, name, serializer) {
		const key = request + "/" + name;
		if (serializers.has(Constructor)) {
			throw new Error(
				`ObjectMiddleware.register: serializer for ${
					Constructor.name
				} is already registered`
			);
		}
		if (serializerInversed.has(key)) {
			throw new Error(
				`ObjectMiddleware.register: serializer for ${key} is already registered`
			);
		}
		serializers.set(Constructor, {
			request,
			name,
			serializer
		});
		serializerInversed.set(key, serializer);
	}

	static getSerializerFor(object) {
		const c = object.constructor;
		const config = serializers.get(c);
		if (!config) throw new Error(`No serializer registered for ${c.name}`);
		return config;
	}

	static getDeserializerFor(request, name) {
		const key = request + "/" + name;
		const serializer = serializerInversed.get(key);
		if (serializer === undefined) {
			throw new Error(`No deserializer registered for ${key}`);
		}
		return serializer;
	}

	_handleFunctionSerialization(fn, context) {
		return () => {
			const r = fn();
			if (r instanceof Promise)
				return r.then(data => this.serialize([data], context));
			return this.serialize([r], context);
		};
	}

	_handleFunctionDeserialization(fn, context) {
		return () => {
			const r = fn();
			if (r instanceof Promise)
				return r.then(data => this.deserialize(data, context)[0]);
			return this.deserialize(r, context)[0];
		};
	}

	/**
	 * @param {any[]} data data items
	 * @param {TODO} context TODO
	 * @returns {any[]|Promise<any[]>} serialized data
	 */
	serialize(data, context) {
		/** @type {any[]} */
		const result = [CURRENT_VERSION];
		let currentPos = 0;
		const referenceable = new Map();
		const addReferenceable = item => {
			referenceable.set(item, currentPos++);
		};
		let currentPosTypeLookup = 0;
		const objectTypeLookup = new Map();
		const cycleStack = new Set();
		const process = item => {
			// check if we can emit a reference
			const ref = referenceable.get(item);
			if (ref !== undefined) {
				result.push(ESCAPE, ref - currentPos);
				return;
			}
			if (typeof item === "object" && item !== null) {
				if (cycleStack.has(item)) {
					throw new Error(
						`Circular references can't be serialized (${Array.from(cycleStack)
							.concat([item])
							.map(obj => obj.constructor.name)
							.join(" -> ")})`
					);
				}
				const { request, name, serializer } = ObjectMiddleware.getSerializerFor(
					item
				);

				const key = `${request}/${name}`;
				const lastIndex = objectTypeLookup.get(key);
				if (lastIndex === undefined) {
					objectTypeLookup.set(key, currentPosTypeLookup++);
					result.push(ESCAPE, request, name);
				} else {
					result.push(ESCAPE, currentPosTypeLookup - lastIndex);
				}
				cycleStack.add(item);
				serializer.serialize(item, {
					write(value) {
						process(value);
					}
				});
				cycleStack.delete(item);
				result.push(ESCAPE, ESCAPE_END_OBJECT);
				addReferenceable(item);
			} else if (typeof item === "string") {
				if (item !== "") {
					// empty strings are shorter when not emitting a reference (this saves 1 byte per empty string)
					addReferenceable(item);
				}
				result.push(item);
			} else if (Buffer.isBuffer(item)) {
				addReferenceable(item);
				result.push(item);
			} else if (item === ESCAPE) {
				result.push(ESCAPE, ESCAPE_ESCAPE_VALUE);
			} else if (typeof item === "function") {
				result.push(this._handleFunctionSerialization(item));
			} else if (item === undefined) {
				result.push(ESCAPE, ESCAPE_UNDEFINED);
			} else {
				result.push(item);
			}
		};
		for (const item of data) {
			process(item);
		}
		return result;
	}

	/**
	 * @param {any[]} data data items<commit_msg>fix serializing of Buffers<commit_after>/*
	MIT License http://www.opensource.org/licenses/mit-license.php
*/

"use strict";

const MapObjectSerializer = require("./MapObjectSerializer");
const PlainObjectSerializer = require("./PlainObjectSerializer");
const SerializerMiddleware = require("./SerializerMiddleware");
const SetObjectSerializer = require("./SetObjectSerializer");

/** @typedef {new (...params: any[]) => any} Constructor */

/*

Format:

File -> Section*
Section -> ObjectSection | ReferenceSection | EscapeSection | OtherSection

ObjectSection -> ESCAPE (
	number:relativeOffset (number > 0) |
	string:request (string|null):export
) Section:value* ESCAPE ESCAPE_END_OBJECT
ReferenceSection -> ESCAPE number:relativeOffset (number < 0)
EscapeSection -> ESCAPE ESCAPE_ESCAPE_VALUE (escaped value ESCAPE)
EscapeSection -> ESCAPE ESCAPE_UNDEFINED (escaped value ESCAPE)
OtherSection -> any (except ESCAPE)

Why using null as escape value?
Multiple null values can merged by the BinaryMiddleware, which makes it very efficient
Technically any value can be used.

*/

/**
 * @typedef {Object} ObjectSerializerContext
 * @property {function(any): void} write
 */

/**
 * @typedef {Object} ObjectDeserializerContext
 * @property {function(): any} read
 */

/**
 * @typedef {Object} ObjectSerializer
 * @property {function(any, ObjectSerializerContext): void} serialize
 * @property {function(ObjectDeserializerContext): any} deserialize
 */

const ESCAPE = null;
const ESCAPE_ESCAPE_VALUE = null;
const ESCAPE_END_OBJECT = true;
const ESCAPE_UNDEFINED = false;

const CURRENT_VERSION = 1;

const plainObjectSerializer = new PlainObjectSerializer();
const mapObjectSerializer = new MapObjectSerializer();
const setObjectSerializer = new SetObjectSerializer();

const serializers = new Map();
const serializerInversed = new Map();

const loadedRequests = new Set();

serializers.set(Object, {
	request: "",
	name: null,
	serializer: plainObjectSerializer
});
serializers.set(Array, {
	request: "",
	name: null,
	serializer: plainObjectSerializer
});
serializers.set(Map, {
	request: "",
	name: 1,
	serializer: mapObjectSerializer
});
serializers.set(Set, {
	request: "",
	name: 2,
	serializer: setObjectSerializer
});
for (const { request, name, serializer } of serializers.values()) {
	serializerInversed.set(`${request}/${name}`, serializer);
}

class ObjectMiddleware extends SerializerMiddleware {
	/**
	 * @param {Constructor} Constructor the constructor
	 * @param {string} request the request which will be required when deserializing
	 * @param {string} name the name to make multiple serializer unique when sharing a request
	 * @param {ObjectSerializer} serializer the serializer
	 * @returns {void}
	 */
	static register(Constructor, request, name, serializer) {
		const key = request + "/" + name;
		if (serializers.has(Constructor)) {
			throw new Error(
				`ObjectMiddleware.register: serializer for ${
					Constructor.name
				} is already registered`
			);
		}
		if (serializerInversed.has(key)) {
			throw new Error(
				`ObjectMiddleware.register: serializer for ${key} is already registered`
			);
		}
		serializers.set(Constructor, {
			request,
			name,
			serializer
		});
		serializerInversed.set(key, serializer);
	}

	static getSerializerFor(object) {
		const c = object.constructor;
		const config = serializers.get(c);
		if (!config) throw new Error(`No serializer registered for ${c.name}`);
		return config;
	}

	static getDeserializerFor(request, name) {
		const key = request + "/" + name;
		const serializer = serializerInversed.get(key);
		if (serializer === undefined) {
			throw new Error(`No deserializer registered for ${key}`);
		}
		return serializer;
	}

	_handleFunctionSerialization(fn, context) {
		return () => {
			const r = fn();
			if (r instanceof Promise)
				return r.then(data => this.serialize([data], context));
			return this.serialize([r], context);
		};
	}

	_handleFunctionDeserialization(fn, context) {
		return () => {
			const r = fn();
			if (r instanceof Promise)
				return r.then(data => this.deserialize(data, context)[0]);
			return this.deserialize(r, context)[0];
		};
	}

	/**
	 * @param {any[]} data data items
	 * @param {TODO} context TODO
	 * @returns {any[]|Promise<any[]>} serialized data
	 */
	serialize(data, context) {
		/** @type {any[]} */
		const result = [CURRENT_VERSION];
		let currentPos = 0;
		const referenceable = new Map();
		const addReferenceable = item => {
			referenceable.set(item, currentPos++);
		};
		let currentPosTypeLookup = 0;
		const objectTypeLookup = new Map();
		const cycleStack = new Set();
		const process = item => {
			// check if we can emit a reference
			const ref = referenceable.get(item);
			if (ref !== undefined) {
				result.push(ESCAPE, ref - currentPos);
				return;
			}
			if (Buffer.isBuffer(item)) {
				addReferenceable(item);
				result.push(item);
			} else if (typeof item === "object" && item !== null) {
				if (cycleStack.has(item)) {
					throw new Error(
						`Circular references can't be serialized (${Array.from(cycleStack)
							.concat([item])
							.map(obj => obj.constructor.name)
							.join(" -> ")})`
					);
				}
				const { request, name, serializer } = ObjectMiddleware.getSerializerFor(
					item
				);

				const key = `${request}/${name}`;
				const lastIndex = objectTypeLookup.get(key);
				if (lastIndex === undefined) {
					objectTypeLookup.set(key, currentPosTypeLookup++);
					result.push(ESCAPE, request, name);
				} else {
					result.push(ESCAPE, currentPosTypeLookup - lastIndex);
				}
				cycleStack.add(item);
				serializer.serialize(item, {
					write(value) {
						process(value);
					}
				});
				cycleStack.delete(item);
				result.push(ESCAPE, ESCAPE_END_OBJECT);
				addReferenceable(item);
			} else if (typeof item === "string") {
				if (item !== "") {
					// empty strings are shorter when not emitting a reference (this saves 1 byte per empty string)
					addReferenceable(item);
				}
				result.push(item);
			} else if (item === ESCAPE) {
				result.push(ESCAPE, ESCAPE_ESCAPE_VALUE);
			} else if (typeof item === "function") {
				result.push(this._handleFunctionSerialization(item));
			} else if (item === undefined) {
				result.push(ESCAPE, ESCAPE_UNDEFINED);
			} else {
				result.push(item);
			}
		};
		for (const item of data) {
			process(item);
		}
		return result;
	}

	/**
	 * @param {any[]} data data items
{'fraction_non_alphanumeric': 0.13320869362000468, 'fraction_numerical': 0.001557996416608242, 'mean_word_length': 4.0031176929072485, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5884247', 'n_tokens_mistral': 4508, 'n_tokens_neox': 3851, 'n_words': 1042}