<commit_before>//! Eudex is a Soundex-esque phonetic reduction/hashing algorithm, providing locality sensitive
//! "hashes" of words, based on the spelling and pronunciation.

#![cfg_attr(test, feature(test))]
#[cfg(test)]
extern crate test;

/// The sound table.
///
/// The first bit each describes a certain property of the phone:
///
/// | Position | Modifier | Property     | Phones                   |
/// |----------|---------:|--------------|:------------------------:|
/// | 1        | 1        | Discriminant | (for tagging duplicates) |
/// | 2        | 2        | Nasal        | mn                       |
/// | 3        | 4        | Fricative    | fvsjxzhct                |
/// | 4        | 8        | Plosive      | pbtdcgqk                 |
/// | 5        | 16       | Dental       | tdnzs                    |
/// | 6        | 32       | Liquid       | lr                       |
/// | 7        | 64       | Labial       | bfpv                     |
/// | 8        | 128      | Confident¹   | lrxzq                    |
///
/// ¹hard to misspell.
///
/// Vowels are, to maxize the XOR distance, represented by 0 and 1 (open and close, respectively).
const PHONES: [u64; LETTERS as usize] = [
    0, // a
    0b01001000, // b
    0b00001100, // c
    0b00011000, // d
    0, // e
    0b01000100, // f
    0b00001000, // g
    0b00000100, // h
    1, // i
    0b00000101, // j
    0b00001001, // k
    0b10100000, // l
    0b00000010, // m
    0b00010010, // n
    0, // o
    0b01001001, // p
    0b10101000, // q
    0b10100001, // r
    0b00010100, // s
    0b00011101, // t
    1, // u
    0b01000101, // v
    0b00000000, // w
    0b10000100, // x
    1, // y
    0b10010100, // z
];

// Non ASCII-phones.
//
// Starts 0xDF (ß). These are all aproixmated sounds, since they can vary a lot between languages.
const PHONES_C1: [u64; 33] = [
    PHONES[(b's' - b'a') as usize] ^ 1, // ß
    PHONES[(b'a' - b'a') as usize], // à
    PHONES[(b'a' - b'a') as usize], // á
    PHONES[(b'a' - b'a') as usize], // â
    PHONES[(b'a' - b'a') as usize], // ã
    PHONES[(b'e' - b'a') as usize], // ä [æ]
    PHONES[(b'o' - b'a') as usize], // å [oː]
    PHONES[(b'e' - b'a') as usize], // æ [æ]
    PHONES[(b'z' - b'a') as usize] ^ 1, // ç [t͡ʃ]
    PHONES[(b'e' - b'a') as usize], // è
    PHONES[(b'e' - b'a') as usize], // é
    PHONES[(b'e' - b'a') as usize], // ê
    PHONES[(b'e' - b'a') as usize], // ë
    PHONES[(b'i' - b'a') as usize], // ì
    PHONES[(b'i' - b'a') as usize], // í
    PHONES[(b'i' - b'a') as usize], // î
    PHONES[(b'i' - b'a') as usize], // ï
    PHONES[(b't' - b'a') as usize] ^ 8, // ð [ð̠] (represented as a non-plosive T)
    PHONES[(b'n' - b'a') as usize] | PHONES[(b'j' - b'a') as usize], // ñ [nj] (represented as a combination of n and j)
    PHONES[(b'o' - b'a') as usize], // ò
    PHONES[(b'o' - b'a') as usize], // ó
    PHONES[(b'o' - b'a') as usize], // ô
    PHONES[(b'o' - b'a') as usize], // õ
    PHONES[(b'o' - b'a') as usize], //) ö
    !0, // ÷
    PHONES[(b'o' - b'a') as usize], // ø
    PHONES[(b'u' - b'a') as usize], // ù
    PHONES[(b'u' - b'a') as usize], // ú
    PHONES[(b'u' - b'a') as usize], // û
    PHONES[(b'y' - b'a') as usize], // ü
    PHONES[(b'y' - b'a') as usize], // ý
    PHONES[(b't' - b'a') as usize] ^ 8, // þ [ð̠] (represented as a non-plosive T)
    PHONES[(b'i' - b'a') as usize], // ÿ
];

/// An _injective_ phone table.
///
/// The first bit (MSD) is set if it is a vowel. If so, the second bit represent, if it is close
/// or not, and the third is set, if it is a front vowel. The rest of the digits are used as
/// discriminants.
///
/// If it is a consonant, the rest of the bits are simply a right truncated version of the
/// [`PHONES`](./const.PHONES.hmtl) table, with the LSD used as discriminant.
///
/// The discriminants of the vowels are chosen in the following way:
///
/// 1. 00000001 (double each time)
/// 2. 00000010
/// 3. 00000100
/// 3. 00001000
/// 4. 00010000
/// 5. 00000011 (set the LSD and use the same pattern as above, skip the first step)
/// 6. 00000101
/// 7. 00001001
/// 8. 00010001
/// 9. 00000110 (use 2 to mark the next section)
/// 10. 00001010
/// 11. 00010010
const INJECTIVE_PHONES: [u64; LETTERS as usize] = [
    0b10100000, // a
    0b00100100, // b
    0b00000110, // c
    0b00001100, // d
    0b11100000, // e
    0b00100010, // f
    0b00000100, // g
    0b00000010, // h
    0b11100001, // i
    0b00000011, // j
    0b00000101, // k
    0b01010000, // l
    0b00000001, // m
    0b00001001, // n
    0b10000000, // o
    0b00100101, // p
    0b01010100, // q
    0b01010001, // r
    0b00001010, // s
    0b00001110, // t
    0b11000001, // u
    0b00100011, // v
    0b00000000, // w
    0b01000010, // x
    0b11100010, // y
    0b01001010, // z
];

/// Non-ASCII injective phone table.
///
/// Starting at C1.
const INJECTIVE_PHONES_C1: [u64; 33] = [
    INJECTIVE_PHONES[(b's' - b'a') as usize] ^ 1, // ß
    INJECTIVE_PHONES[(b'a' - b'a') as usize] ^ 2, // à
    INJECTIVE_PHONES[(b'a' - b'a') as usize] ^ 4, // á
    INJECTIVE_PHONES[(b'a' - b'a') as usize] ^ 8, // â
    INJECTIVE_PHONES[(b'a' - b'a') as usize] ^ 16, // ã
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 4, // ä [æ]
    INJECTIVE_PHONES[(b'o' - b'a') as usize] ^ 1, // å [oː]
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 8, // æ [æ]
    INJECTIVE_PHONES[(b'z' - b'a') as usize] ^ 1, // ç [t͡ʃ]
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 16, // è
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 3, // é
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 5, // ê
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 7, // ë
    INJECTIVE_PHONES[(b'i' - b'a') as usize] ^ 17, // ì<commit_msg>A new scheme for deriving vowel representation for the first slot<commit_after>//! Eudex is a Soundex-esque phonetic reduction/hashing algorithm, providing locality sensitive
//! "hashes" of words, based on the spelling and pronunciation.

#![cfg_attr(test, feature(test))]
#[cfg(test)]
extern crate test;

/// The sound table.
///
/// The first bit each describes a certain property of the phone:
///
/// | Position | Modifier | Property     | Phones                   |
/// |----------|---------:|--------------|:------------------------:|
/// | 1        | 1        | Discriminant | (for tagging duplicates) |
/// | 2        | 2        | Nasal        | mn                       |
/// | 3        | 4        | Fricative    | fvsjxzhct                |
/// | 4        | 8        | Plosive      | pbtdcgqk                 |
/// | 5        | 16       | Dental       | tdnzs                    |
/// | 6        | 32       | Liquid       | lr                       |
/// | 7        | 64       | Labial       | bfpv                     |
/// | 8        | 128      | Confident¹   | lrxzq                    |
///
/// ¹hard to misspell.
///
/// Vowels are, to maxize the XOR distance, represented by 0 and 1 (open and close, respectively).
const PHONES: [u64; LETTERS as usize] = [
    0, // a
    0b01001000, // b
    0b00001100, // c
    0b00011000, // d
    0, // e
    0b01000100, // f
    0b00001000, // g
    0b00000100, // h
    1, // i
    0b00000101, // j
    0b00001001, // k
    0b10100000, // l
    0b00000010, // m
    0b00010010, // n
    0, // o
    0b01001001, // p
    0b10101000, // q
    0b10100001, // r
    0b00010100, // s
    0b00011101, // t
    1, // u
    0b01000101, // v
    0b00000000, // w
    0b10000100, // x
    1, // y
    0b10010100, // z
];

// Non ASCII-phones.
//
// Starts 0xDF (ß). These are all aproixmated sounds, since they can vary a lot between languages.
const PHONES_C1: [u64; 33] = [
    PHONES[(b's' - b'a') as usize] ^ 1, // ß
    PHONES[(b'a' - b'a') as usize], // à
    PHONES[(b'a' - b'a') as usize], // á
    PHONES[(b'a' - b'a') as usize], // â
    PHONES[(b'a' - b'a') as usize], // ã
    PHONES[(b'e' - b'a') as usize], // ä [æ]
    PHONES[(b'o' - b'a') as usize], // å [oː]
    PHONES[(b'e' - b'a') as usize], // æ [æ]
    PHONES[(b'z' - b'a') as usize] ^ 1, // ç [t͡ʃ]
    PHONES[(b'e' - b'a') as usize], // è
    PHONES[(b'e' - b'a') as usize], // é
    PHONES[(b'e' - b'a') as usize], // ê
    PHONES[(b'e' - b'a') as usize], // ë
    PHONES[(b'i' - b'a') as usize], // ì
    PHONES[(b'i' - b'a') as usize], // í
    PHONES[(b'i' - b'a') as usize], // î
    PHONES[(b'i' - b'a') as usize], // ï
    PHONES[(b't' - b'a') as usize] ^ 8, // ð [ð̠] (represented as a non-plosive T)
    PHONES[(b'n' - b'a') as usize] | PHONES[(b'j' - b'a') as usize], // ñ [nj] (represented as a combination of n and j)
    PHONES[(b'o' - b'a') as usize], // ò
    PHONES[(b'o' - b'a') as usize], // ó
    PHONES[(b'o' - b'a') as usize], // ô
    PHONES[(b'o' - b'a') as usize], // õ
    PHONES[(b'o' - b'a') as usize], //) ö
    !0, // ÷
    PHONES[(b'o' - b'a') as usize], // ø
    PHONES[(b'u' - b'a') as usize], // ù
    PHONES[(b'u' - b'a') as usize], // ú
    PHONES[(b'u' - b'a') as usize], // û
    PHONES[(b'y' - b'a') as usize], // ü
    PHONES[(b'y' - b'a') as usize], // ý
    PHONES[(b't' - b'a') as usize] ^ 8, // þ [ð̠] (represented as a non-plosive T)
    PHONES[(b'i' - b'a') as usize], // ÿ
];

/// An _injective_ phone table.
///
/// The table is derived the following way:
///
/// | Position | Modifier | Property (vowel)    | Property (consonant)                              |
/// |----------|---------:|---------------------|---------------------------------------------------|
/// | 1        | 1        | Discriminant        | (property 2 from the phone table) or discriminant |
/// | 4        | 8        | Is it open-mid?     | (property 3 from the phone table)                 |
/// | 2        | 2        | Is it central?      | (property 4 from the phone table)                 |
/// | 5        | 16       | Is it close-mid?    | (property 5 from the phone table)                 |
/// | 3        | 4        | Is it front?        | (property 6 from the phone table)                 |
/// | 6        | 32       | Is it close?        | (property 7 from the phone table)                 |
/// | 7        | 64       | More close than [ɜ] | (property 8 from the phone table)                 |
/// | 8        | 128      | Vowel?                                                                  |
///
/// If it is a consonant, the rest of the bits are simply a right truncated version of the
/// [`PHONES`](./const.PHONES.html) table, with the LSD used as discriminant.
const INJECTIVE_PHONES: [u64; LETTERS as usize] = [
    0b10000110, // a
    0b00100100, // b
    0b00000110, // c
    0b00001100, // d
    0b11011100, // e
    0b00100010, // f
    0b00000100, // g
    0b00000010, // h
    0b11111000, // i
    0b00000011, // j
    0b00000101, // k
    0b01010000, // l
    0b00000001, // m
    0b00001001, // n
    0b11001100, // o
    0b00100101, // p
    0b01010100, // q
    0b01010001, // r
    0b00001010, // s
    0b00001110, // t
    0b11100000, // u
    0b00100011, // v
    0b00000000, // w
    0b01000010, // x
    0b11100100, // y
    0b01001010, // z
];

/// Non-ASCII injective phone table.
///
/// Starting at C1.
const INJECTIVE_PHONES_C1: [u64; 33] = [
    INJECTIVE_PHONES[(b's' - b'a') as usize] ^ 1, // ß
    INJECTIVE_PHONES[(b'a' - b'a') as usize] ^ 2, // à
    INJECTIVE_PHONES[(b'a' - b'a') as usize] ^ 4, // á
    INJECTIVE_PHONES[(b'a' - b'a') as usize] ^ 8, // â
    INJECTIVE_PHONES[(b'a' - b'a') as usize] ^ 16, // ã
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 4, // ä [æ]
    INJECTIVE_PHONES[(b'o' - b'a') as usize] ^ 1, // å [oː]
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 8, // æ [æ]
    INJECTIVE_PHONES[(b'z' - b'a') as usize] ^ 1, // ç [t͡ʃ]
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 16, // è
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 3, // é
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 5, // ê
    INJECTIVE_PHONES[(b'e' - b'a') as usize] ^ 7, // ë
    INJECTIVE_PHONES[(b'i' - b'a') as usize] ^ 17, // ì
{'fraction_non_alphanumeric': 0.20318925723877465, 'fraction_numerical': 0.09240453210239194, 'mean_word_length': 1.5620296710384864, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3127046', 'n_tokens_mistral': 5574, 'n_tokens_neox': 4809, 'n_words': 1323}