<commit_before>#[macro_export]
macro_rules! char_class {
    ($c: expr, [$head:expr]) => ($c == $head);
    ($c: expr, [$head:expr $(, $cs:expr)+]) => ($c == $head || $crate::char_class!($c, [$($cs),*]));
}

#[macro_export]
macro_rules! alpha_char {
    ($c: expr) => {
        match $c {
            'a'..='z' => true,
            'A'..='Z' => true,
            '_' => true,
            '\u{00A0}'..='\u{00BF}' => true,
            '\u{00C0}'..='\u{00D6}' => true,
            '\u{00D8}'..='\u{00F6}' => true,
            '\u{00F8}'..='\u{00FF}' => true,
            '\u{0100}'..='\u{017F}' => true, // Latin Extended-A
            '\u{0180}'..='\u{024F}' => true, // Latin Extended-B
            '\u{0250}'..='\u{02AF}' => true, // IPA Extensions
            '\u{02B0}'..='\u{02FF}' => true, // Spacing Modifier Letters
            '\u{0300}'..='\u{036F}' => true, // Combining Diacritical Marks
            '\u{0370}'..='\u{03FF}' => true, // Greek/Coptic
            '\u{0400}'..='\u{04FF}' => true, // Cyrillic
            '\u{0500}'..='\u{052F}' => true, // Cyrillic Supplement
            '\u{0530}'..='\u{058F}' => true, // Armenian
            '\u{0590}'..='\u{05FF}' => true, // Hebrew
            '\u{0600}'..='\u{06FF}' => true, // Arabic
            '\u{0700}'..='\u{074F}' => true, // Syriac
            _ => false,
        }
    };
}

#[macro_export]
macro_rules! alpha_numeric_char {
    ($c: expr) => {
        $crate::alpha_char!($c) || $crate::decimal_digit_char!($c)
    };<commit_msg>accept remaining UTF-8 codepoints as alphabetic characters (#1355)<commit_after>#[macro_export]
macro_rules! char_class {
    ($c: expr, [$head:expr]) => ($c == $head);
    ($c: expr, [$head:expr $(, $cs:expr)+]) => ($c == $head || $crate::char_class!($c, [$($cs),*]));
}

#[macro_export]
macro_rules! alpha_char {
    ($c: expr) => {
        match $c {
            'a'..='z' => true,
            'A'..='Z' => true,
            '_' => true,
            '\u{00A0}'..='\u{00BF}' => true,
            '\u{00C0}'..='\u{00D6}' => true,
            '\u{00D8}'..='\u{00F6}' => true,
            '\u{00F8}'..='\u{00FF}' => true,
            '\u{0100}'..='\u{FFFF}' => true, /* skip some control
 characters but admit the remaining code points as alphabetic
 characters. */
            _ => false,
        }
    };
}

#[macro_export]
macro_rules! alpha_numeric_char {
    ($c: expr) => {
        $crate::alpha_char!($c) || $crate::decimal_digit_char!($c)
    };
{'fraction_non_alphanumeric': 0.27024827024827025, 'fraction_numerical': 0.05291005291005291, 'mean_word_length': 2.023370233702337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3786844', 'n_tokens_mistral': 1028, 'n_tokens_neox': 944, 'n_words': 154}