<commit_before>/*
 * PurePO2JSON v3.0.0
 * by <NAME> (An_dz)
 *
 * with previous contributions by <NAME> (QuHno)
 * IBM i18n idea and formula from <NAME> (QuHno)
 */
"use strict";

/**
 * @brief Convert special characters to their unicode number
 *
 * Used by the `replace` method to convert the characters to
 * their unicode values between underlines.
 */
function specialChars(match, linefeed, special) {
	// get rid of new lines
	if (linefeed === "n") {
		return "_10_";
	}
	if (linefeed === "r") {
		return "_13_";
	}
	// convert special chars to their code
	if (special !== undefined) {
		return "_" + match.charCodeAt(0) + "_";
	}
	return match;
}

/**
 * @brief Class representing a message
 *
 * This class is a construction for one message, it holds all
 * the data about the message, including original text, context
 * and all possible translations (if there are plurals)
 *
 * @warning: The setters append and don't replace
 */
class Message {
	/**
	 * @brief Construct a new message
	 *
	 * @param lf {string} The linefeed for each line,
	 * minimised JSON if left as an empty string
	 */
	constructor(lf, ibmi18n) {
		this.linefeed = lf;
		this.ibmi18n  = ibmi18n;
		this.tab      = (lf === "" ? "" : "\t");
		this.space    = (lf === "" ? "" : " ");
		this.str      = [];
		this.id       = "";
		this.ctxt     = "";
		this.lastUsed = null;
	}
	/**
	 * @brief Set or amend msgid
	 *
	 * @param text {string} The text to set
	 */
	set msgid(text) {
		this.id = text;
		this.lastUsed = this.id;
	}
	get msgid() {
		return this.id;
	}
	/**
	 * @brief Set the context
	 *
	 * @param context {string} The text to set
	 */
	set msgctxt(context) {
		this.ctxt = context;
		this.lastUsed = this.ctxt;
	}
	get msgctxt() {
		return this.ctxt;
	}
	/**
	 * @brief Set or amend msgid
	 *
	 * @param string {string} The text to set
	 */
	set msgstr(string) {
		const index = this.str.push(string) - 1;
		this.lastUsed = this.str[index];
	}
	get msgstr() {
		return this.str;
	}
	/**
	 * @brief Appends text to the last defined property
	 *
	 * @param string {string} The text to append
	 */
	set amend(string) {
		this.lastUsed += string;
	}
	/**
	 * @brief Increase string to safe length if enabled
	 *
	 * Applies the IBM algorithm that increases the length of the
	 * string to test screen space in multiple languages.
	 *
	 * @param string {string} the text to apply IBM algorithm
	 *
	 * @return {string} String with increase length or same string
	 */
	getIBM(string) {
		if (this.ibmi18n !== true) {
			return string;
		}

		const length = string.length;

		return `${string}${"-".repeat(
			(3 / Math.log(length) + 0.7) * length - length
		)}`;
	}
	/**
	 * @brief Return a constructed JSON as a string
	 *
	 * @return {string} JSON of the message
	 */
	toString() {
		const lf = this.linefeed;
		const tab = this.tab;
		const space = this.space;

		const msgctxt = (this.msgctxt.length > 0 ? "\x04" : "");

		const id = `${this.msgctxt}${msgctxt}${this.msgid}`.replace(
			/\\(n|r)|([^a-z0-9])/g, specialChars
		);

		const json = [];

		this.msgstr.forEach((string, index) => {
			if (string.length === 0) {
				string = this.msgid;
			}

			json.push(
				`${id}${index}:${space}{${lf}` +
					`${tab}"message":${space}"${this.getIBM(string)}"${lf}` +
				"}"
			);
		});

		return json.join(`,${lf}`);
	}
}<commit_msg>Fix missing quotes around id key<commit_after>/*
 * PurePO2JSON v3.0.0
 * by <NAME> (An_dz)
 *
 * with previous contributions by <NAME> (QuHno)
 * IBM i18n idea and formula from <NAME> (QuHno)
 */
"use strict";

/**
 * @brief Convert special characters to their unicode number
 *
 * Used by the `replace` method to convert the characters to
 * their unicode values between underlines.
 */
function specialChars(match, linefeed, special) {
	// get rid of new lines
	if (linefeed === "n") {
		return "_10_";
	}
	if (linefeed === "r") {
		return "_13_";
	}
	// convert special chars to their code
	if (special !== undefined) {
		return "_" + match.charCodeAt(0) + "_";
	}
	return match;
}

/**
 * @brief Class representing a message
 *
 * This class is a construction for one message, it holds all
 * the data about the message, including original text, context
 * and all possible translations (if there are plurals)
 *
 * @warning: The setters append and don't replace
 */
class Message {
	/**
	 * @brief Construct a new message
	 *
	 * @param lf {string} The linefeed for each line,
	 * minimised JSON if left as an empty string
	 */
	constructor(lf, ibmi18n) {
		this.linefeed = lf;
		this.ibmi18n  = ibmi18n;
		this.tab      = (lf === "" ? "" : "\t");
		this.space    = (lf === "" ? "" : " ");
		this.str      = [];
		this.id       = "";
		this.ctxt     = "";
		this.lastUsed = null;
	}
	/**
	 * @brief Set or amend msgid
	 *
	 * @param text {string} The text to set
	 */
	set msgid(text) {
		this.id = text;
		this.lastUsed = this.id;
	}
	get msgid() {
		return this.id;
	}
	/**
	 * @brief Set the context
	 *
	 * @param context {string} The text to set
	 */
	set msgctxt(context) {
		this.ctxt = context;
		this.lastUsed = this.ctxt;
	}
	get msgctxt() {
		return this.ctxt;
	}
	/**
	 * @brief Set or amend msgid
	 *
	 * @param string {string} The text to set
	 */
	set msgstr(string) {
		const index = this.str.push(string) - 1;
		this.lastUsed = this.str[index];
	}
	get msgstr() {
		return this.str;
	}
	/**
	 * @brief Appends text to the last defined property
	 *
	 * @param string {string} The text to append
	 */
	set amend(string) {
		this.lastUsed += string;
	}
	/**
	 * @brief Increase string to safe length if enabled
	 *
	 * Applies the IBM algorithm that increases the length of the
	 * string to test screen space in multiple languages.
	 *
	 * @param string {string} the text to apply IBM algorithm
	 *
	 * @return {string} String with increase length or same string
	 */
	getIBM(string) {
		if (this.ibmi18n !== true) {
			return string;
		}

		const length = string.length;

		return `${string}${"-".repeat(
			(3 / Math.log(length) + 0.7) * length - length
		)}`;
	}
	/**
	 * @brief Return a constructed JSON as a string
	 *
	 * @return {string} JSON of the message
	 */
	toString() {
		const lf = this.linefeed;
		const tab = this.tab;
		const space = this.space;

		const msgctxt = (this.msgctxt.length > 0 ? "\x04" : "");

		const id = `${this.msgctxt}${msgctxt}${this.msgid}`.replace(
			/\\(n|r)|([^a-z0-9])/g, specialChars
		);

		const json = [];

		this.msgstr.forEach((string, index) => {
			if (string.length === 0) {
				string = this.msgid;
			}

			json.push(
				`"${id}${index}":${space}{${lf}` +
					`${tab}"message":${space}"${this.getIBM(string)}"${lf}` +
				"}"
			);
		});

		return json.join(`,${lf}`);
	}
}
{'fraction_non_alphanumeric': 0.15750860907321454, 'fraction_numerical': 0.008683934720766583, 'mean_word_length': 2.856812933025404, 'pattern_counts': {'":': 3, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '452811', 'n_tokens_mistral': 2554, 'n_tokens_neox': 2294, 'n_words': 705}