<commit_before>Orth,Phon
b,p
p,pʰ
m,m
f,f
d,t
t,tʰ
n,n
l,l
g,k
k,kʰ
ng,ŋ
h,h
gw,kʷ
kw,kʷʰ
w,w
z,t͡s
c,t͡sʰ
s,s
j,j
aa,aː
aai,aːj
aau,aːw
a,ɐ
e,ɛː
ei,ej
eu,ɛːw
i,iː
ing,ɪŋ
ik,ɪk
o,ɔː
oi,ɔːɥ
ou,ow
u,uː
ui,uːɥ
ung,ʊŋ
uk,ʊk
eo,ɵ
eoi,ɵɥ
oe,œː
yu,yː
1,˥
2,˧˥
3,˧
4,˨˩
5,˩˧
6,˨<commit_msg>yue: for diphthongs, switch from phonological (vowel + glide / glide + vowel) to phonetic (2 vowels) representation<commit_after>Orth,Phon
b,p
p,pʰ
m,m
f,f
d,t
t,tʰ
n,n
l,l
g,k
k,kʰ
ng,ŋ
h,h
gw,kʷ
kw,kʷʰ
w,w
z,t͡s
c,t͡sʰ
s,s
j,j
aa,aː
aai,aːi
aau,aːu
a,ɐ
e,ɛː
ei,ei
eu,ɛːu
i,iː
ing,ɪŋ
ik,ɪk
o,ɔː
oi,ɔːi
ou,ou
u,uː
ui,uːi
ung,ʊŋ
uk,ʊk
eo,ɵ
eoi,ɵi
oe,œː
yu,yː
1,˥
2,˧˥
3,˧
4,˨˩
5,˩˧
6,˨

{'fraction_non_alphanumeric': 0.19640179910044978, 'fraction_numerical': 0.019490254872563718, 'mean_word_length': 4.964285714285714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3686411', 'n_tokens_mistral': 540, 'n_tokens_neox': 602, 'n_words': 107}