<commit_before>export const defs = {
	nbsp: '\xA0',
	hairspace: '\xAF',
	space: ({ nbsp, hairspace }) => `[ \t${nbsp}${hairspace}]`,
	tag: '<[^<>]*>',
	quotes: '["“”«»‘’]',
	letters: '[a-zà-ž0-9]',
	upperletters: '[A-ZÀ-Ž0-9]',
	nonletters: '[^a-zà-ž0-9]',
	letterswithquotes: '[a-zà-ž0-9-“”‘’«»]',
	semicolon: '(?<!&\\S*);',
	punctuation: ({ semicolon }) => `(?:${semicolon}|[\\.,!?:])`,
	dash: '[-—]',
	thousandsSeparator: ',',
	decimalsSeparator: '[.]',
	ordinals: '(?:st|nd|rd|th)',
	openingQuote: '“',
	closingQuote: '”',
	shortWord: ({ letters }) => `${letters}{1,2}`,
};

// Non-breaking space after short words
const shortWordsBreak = ({ shortWord, quotes, space, nbsp, tag }) => ({
	regex: `(?<=^|${space}|${quotes}|>)(${shortWord}(${tag})?)${space}`,
	result: `$1${nbsp}`,
});

// Orphans (non-breaking space before the last word)
export const orphans = ({ space, nbsp }) => ({
	regex: `${space}([\\S<]{1,10}(?:\n\n|$))`,
	result: `${nbsp}$1`,
});

export const numbersUnits = ({ space, nbsp }) => ({
	regex: `(\\d+)${space}(\\w)`,
	result: `$1${nbsp}$2`,
});

export const spaces = [numbersUnits, shortWordsBreak, orphans];

export const emdash = [
	({ space, nbsp }) => ({
		regex: `(\\S)${space}?—`,
		result: `$1${nbsp}—`,
	}),
	{
		regex: `—(\\S)`,
		result: `— $1`,
	},
	({ space, nbsp, letterswithquotes, tag, dash }) => ({
		regex: `(${letterswithquotes}(${tag})?)${space}${dash}`,
		result: `$1${nbsp}—`,
	}),
	({ space, nbsp, punctuation, openingQuote, dash }) => ({
		regex: `(^|(?:(${punctuation}|${openingQuote}|")${space}?))${dash}${space}`,
		result: `$1—${nbsp}`,
	}),
];

export const ellipsis = {
	regex: `\\.{2,}`,
	result: `…`,
};

export const amp = ({ space, nbsp }) => ({
	regex: `${space}(&(?!\\S*;))${space}`,
	result: `${nbsp}<span class="amp">&</span>${nbsp}`,
});

export const abbr = ({ upperletters }) => ({
	regex: `(${upperletters}{3,})`,
	flags: `gm`,
	result: `<abbr>$1</abbr>`,
});

export const numbers = [
	({ ordinals }) => ({
		regex: `(\\d+)(${ordinals})`,
		result: `$1<sup>$2</sup>`,
	}),
	({ decimalsSeparator, thousandsSeparator }) => ({
		regex: `(?<!${decimalsSeparator}\\d*)\\d{1,3}(?=(\\d{3})+(?!\\d))`,
		result: `$&${thousandsSeparator}`,
	}),
];

export const quotes = [
	({ tag, letters, dash, space, openingQuote }) => ({
		regex: `"((${tag})?(${dash}${space})?${letters})`,
		result: `${openingQuote}$1`,
	}),
	({ closingQuote }) => ({
		regex: `"`,
		result: `${closingQuote}`,
	}),
];<commit_msg>Added temperature rule and spaces rule is composed differently<commit_after>export const defs = {
	nbsp: '\xA0',
	hairspace: '\xAF',
	space: ({ nbsp, hairspace }) => `[ \t${nbsp}${hairspace}]`,
	tag: '<[^<>]*>',
	quotes: '["“”«»‘’]',
	letters: '[a-zà-ž0-9]',
	upperletters: '[A-ZÀ-Ž]',
	nonletters: '[^a-zà-ž0-9]',
	letterswithquotes: '[a-zà-ž0-9-“”‘’«»]',
	semicolon: '(?<!&\\S*);',
	punctuation: ({ semicolon }) => `(?:${semicolon}|[\\.,!?:])`,
	dash: '[-—]',
	thousandsSeparator: ',',
	decimalsSeparator: '[.]',
	ordinals: '(?:st|nd|rd|th)',
	openingQuote: '“',
	closingQuote: '”',
	shortWord: ({ letters }) => `${letters}{1,2}`,
};

// Non-breaking space after short words
export const shortWordBreak = ({ shortWord, quotes, space, nbsp, tag }) => ({
	regex: `(?<=^|${space}|${quotes}|>)(${shortWord}(${tag})?)${space}`,
	replace: `$1${nbsp}`,
});

// Orphans (non-breaking space before the last word)
export const orphans = ({ space, nbsp }) => ({
	regex: `${space}([\\S<]{1,10}(?:\n\n|$))`,
	replace: `${nbsp}$1`,
});

export const numberUnits = ({ space, nbsp }) => ({
	regex: `(\\d+)${space}(\\w)`,
	replace: `$1${nbsp}$2`,
});

export const temperature = ({ space, nbsp }) => ({
	regex: `(\\d)${space}?°`,
	replace: `$1${nbsp}°`,
});

export const spaces = { numberUnits, temperature, shortWordBreak, orphans };

export const emdash = [
	({ space, nbsp }) => ({
		regex: `(\\S)${space}?—`,
		replace: `$1${nbsp}—`,
	}),
	{
		regex: `—(\\S)`,
		replace: `— $1`,
	},
	({ space, nbsp, letterswithquotes, tag, dash }) => ({
		regex: `(${letterswithquotes}(${tag})?)${space}${dash}`,
		replace: `$1${nbsp}—`,
	}),
	({ space, nbsp, punctuation, openingQuote, dash }) => ({
		regex: `(^|(?:(${punctuation}|${openingQuote}|")${space}?))${dash}${space}`,
		replace: `$1—${nbsp}`,
	}),
];

export const ellipsis = {
	regex: `\\.{2,}`,
	replace: `…`,
};

export const amp = ({ space, nbsp }) => ({
	regex: `${space}(&(?!\\S*;))${space}`,
	replace: `${nbsp}<span class="amp">&</span>${nbsp}`,
});

export const abbr = ({ upperletters }) => ({
	regex: `(${upperletters}{3,})`,
	flags: `gm`,
	replace: `<abbr>$1</abbr>`,
});

export const numbers = [
	({ ordinals }) => ({
		regex: `(\\d+)(${ordinals})`,
		replace: `$1<sup>$2</sup>`,
	}),
	({ decimalsSeparator, thousandsSeparator }) => ({
		regex: `(?<!${decimalsSeparator}\\d*)\\d{1,3}(?=(\\d{3})+(?!\\d))`,
		replace: `$&${thousandsSeparator}`,
	}),
];

export const quotes = [
	({ tag, letters, dash, space, openingQuote }) => ({
		regex: `"((${tag})?(${dash}${space})?${letters})`,
		replace: `${openingQuote}$1`,
	}),
	({ closingQuote }) => ({
		regex: `"`,
		replace: `${closingQuote}`,
	}),
];
{'fraction_non_alphanumeric': 0.3204149540027403, 'fraction_numerical': 0.011939714229790566, 'mean_word_length': 4.833333333333333, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6835136', 'n_tokens_mistral': 2220, 'n_tokens_neox': 2123, 'n_words': 373}