<commit_before>language,en,av,Avar
language,en,frr,North Frisian
language,en,frs,East Frisian
language,en,fy,West Frisian
language,en,gn,Guaraní
language,en,ilo,Ilocano
language,en,jam,Jamaican Creole
language,en,kky,Guugu Yimidhirr
language,en,kky,Guugu Yimithirr
language,en,ksd,Tolai
language,en,liv,Livonian
language,en,nay,Ngarrindjeri
language,en,nmn,ǃXóõ
language,en,oj,Ojibwe
language,en,pap,Papiamentu
language,en,pms,Piedmontese
language,en,rap,Rapa Nui
language,en,rm,Romansch
language,en,rom,Romani
language,en,ryu,Okinawan
language,en,sl,Slovene
language,en,st,Sesotho
language,en,tvl,Tuvaluan
language,en,twf,Taos
language,en,txb,Tocharian B
language,en,tyv,Tuvan
language,en,vma,Martuthunira
language,en,wym,Vilamovian
language,en,xto,Tocharian A
language,en,zu,isiZulu
language,de,el,Neugriechisch
language,de,la,Lateinisch
language,de,fur,Friaulisch
language,de,gd,Schottisch-Gälisch
language,de,haw,Hawaiianisch
language,de,nds,Plattdeutsch
language,de,nhn,Zentral-Nahuatl
language,de,pa,Panjabi
language,de,pap,Papiamentu
language,de,prg,Prußisch
language,de,vec,Venezianisch
language,de,tvl,Tuvaluisch
language,sh,sh,Srpskohrvatski
language,la,la,Lingua latina
language,ceb,ceb,Bisayâ
language,ceb,ceb,Bisaya
language,lah,lah,لہندا پنجابی
language,bho,bho,भोजपुरी
language,ang,ang,Ænglisc
language,vo,vo,Volapük
language,io,io,Ido
language,rup,rup,armãneashti
language,nv,nv,Diné bizaad<commit_msg>Add some more language names so they work as autonyms<commit_after>language,en,av,Avar
language,en,frr,North Frisian
language,en,frs,East Frisian
language,en,fy,West Frisian
language,en,gn,Guaraní
language,en,ilo,Ilocano
language,en,jam,Jamaican Creole
language,en,kky,Guugu Yimidhirr
language,en,kky,Guugu Yimithirr
language,en,ksd,Tolai
language,en,liv,Livonian
language,en,nay,Ngarrindjeri
language,en,nmn,ǃXóõ
language,en,oj,Ojibwe
language,en,pap,Papiamentu
language,en,pms,Piedmontese
language,en,rap,Rapa Nui
language,en,rm,Romansch
language,en,rom,Romani
language,en,ryu,Okinawan
language,en,sl,Slovene
language,en,st,Sesotho
language,en,tvl,Tuvaluan
language,en,twf,Taos
language,en,txb,Tocharian B
language,en,tyv,Tuvan
language,en,vma,Martuthunira
language,en,wym,Vilamovian
language,en,xto,Tocharian A
language,en,zu,isiZulu
language,de,el,Neugriechisch
language,de,la,Lateinisch
language,de,fur,Friaulisch
language,de,gd,Schottisch-Gälisch
language,de,haw,Hawaiianisch
language,de,nds,Plattdeutsch
language,de,nhn,Zentral-Nahuatl
language,de,pa,Panjabi
language,de,pap,Papiamentu
language,de,prg,Prußisch
language,de,vec,Venezianisch
language,de,tvl,Tuvaluisch
language,sh,sh,Srpskohrvatski
language,la,la,Lingua latina
language,ceb,ceb,Sinugbuanong Binisayâ
language,ceb,ceb,Bisayâ
language,ceb,ceb,Bisaya
language,lah,lah,لہندا پنجابی
language,bho,bho,भोजपुरी
language,ang,ang,Ænglisc
language,vo,vo,Volapük
language,io,io,Ido
language,jbo,jbo,lojban
language,jbo,jbo,lojbau
language,rup,rup,armãneashti
language,nv,nv,Diné bizaad
language,zh-Hant,nan,閩南語
language,zh-Hans,nan,闽南语
language,nan-Latn,nan,Bân-lâm-gú
language,zh-Hant,hak,客家語
language,zh-Hans,hak,客家语
language,ilo,ilo,Ilokano
language,hil,hil,Ilonggo
language,nah,nah,Nāhuatl
language,tpi,tpi,Tok Pisin
{'fraction_non_alphanumeric': 0.11788617886178862, 'fraction_numerical': 0.0, 'mean_word_length': 18.625766871165645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7360460', 'n_tokens_mistral': 1475, 'n_tokens_neox': 1455, 'n_words': 152}