<commit_before># encoding: UTF-8
PHONETICS = {
  'AA' => 'ɑ:',
  'AE' => 'æ',
  'AH' => 'ə',
  'AO' => 'ɒ',
  'AW' => 'ou',
  'AY' => 'əɪ',
  'B' => 'b',
  'CH' => 'tʃ',
  'D' => 'd',
  'DH' => 'ð',
  'EH' => 'ɛ',
  'ER' => 'ər',
  'EY' => 'e',
  'F' => 'f',
  'G' => 'g',
  'HH' => 'h',
  'IH' => 'ɪ',
  'IY' => 'i:',
  'JH' => 'dʒ',
  'K' => 'k',
  'L' => 'l',
  'M' => 'm',
  'N' => 'n',
  'NG' => 'ŋ',
  'OW' => 'o',
  'OY' => 'ɔj',
  'P' => 'p',
  'R' => 'r',
  'S' => 's',
  'SH' => 'ʃ',
  'T' => 't',
  'TH' => 'θ',
  'UH' => 'ʊ',
  'UW' => 'u:',
  'V' => 'v',
  'W' => 'w',
  'Y' => 'j',
  'Z' => 'z',
  'ZH' => 'ʒ',
}<commit_msg>fix some phonetic (not sure)<commit_after># encoding: UTF-8
PHONETICS = {
  'AA' => 'ɑ:',
  'AE' => 'æ',
  'AH' => 'ə',
  'AO' => 'ɔː',
  'AW' => 'aʊ',
  'AY' => 'ʌɪ',
  'B' => 'b',
  'CH' => 'tʃ',
  'D' => 'd',
  'DH' => 'ð',
  'EH' => 'ɛ:',
  'ER' => 'əː',
  'EY' => 'eɪ',
  'F' => 'f',
  'G' => 'g',
  'HH' => 'h',
  'IH' => 'ɪ',
  'IY' => 'i:',
  'JH' => 'dʒ',
  'K' => 'k',
  'L' => 'l',
  'M' => 'm',
  'N' => 'n',
  'NG' => 'ŋ',
  'OW' => 'əʊ',
  'OY' => 'ɔɪ',
  'P' => 'p',
  'R' => 'r',
  'S' => 's',
  'SH' => 'ʃ',
  'T' => 't',
  'TH' => 'θ',
  'UH' => 'ʊ',
  'UW' => 'u:',
  'V' => 'v',
  'W' => 'w',
  'Y' => 'j',
  'Z' => 'z',
  'ZH' => 'ʒ',
}
{'fraction_non_alphanumeric': 0.44247104247104246, 'fraction_numerical': 0.0015444015444015444, 'mean_word_length': 2.1687041564792175, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5979422', 'n_tokens_mistral': 802, 'n_tokens_neox': 811, 'n_words': 168}