<commit_before>const nbsp = '\xA0';
const hairspace = '\xAF';
const space = `[ \t${nbsp}${hairspace}]`;
const tag = '(?:<[^<>]*>)';
const quotes = '["“”«»‘’]';
const letters = '[a-zà-ž0-9]';
const upperletters = '[A-ZÀ-Ž]';
const letterswithquotes = '[a-zà-ž0-9-“”‘’«»]';
const semicolon = '(?<!&\\S*);';
const punctuation = `(?:${semicolon}|[\\.,!?:])`;
const dash = '[-—]';
const openingQuotes = `[“«]`;
const shortWord = `${letters}{1,2}`;
const notInTag = `(?<!<[^>]*)`;

export const definitions = {
	nbsp,
	hairspace,
	space,
	tag,
	quotes,
	letters,
	upperletters,<commit_msg>Added Cyrillic letters<commit_after>const nbsp = '\xA0';
const hairspace = '\xAF';
const space = `[ \t${nbsp}${hairspace}]`;
const tag = '(?:<[^<>]*>)';
const quotes = '["“”«»‘’]';
const letters = '[a-zà-ž0-9а-яё]';
const upperletters = '[A-ZÀ-ŽА-ЯЁ]';
const letterswithquotes = `(?:${letters}|[-“”‘’«»])`;
const semicolon = '(?<!&\\S*);';
const punctuation = `(?:${semicolon}|[\\.,!?:])`;
const dash = '[-—]';
const openingQuotes = `[“«]`;
const shortWord = `${letters}{1,2}`;
const notInTag = `(?<!<[^>]*)`;

export const definitions = {
	nbsp,
	hairspace,
	space,
	tag,
	quotes,
	letters,
	upperletters,
{'fraction_non_alphanumeric': 0.2968881412952061, 'fraction_numerical': 0.010092514718250631, 'mean_word_length': 5.2631578947368425, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '967802', 'n_tokens_mistral': 532, 'n_tokens_neox': 515, 'n_words': 107}