<commit_before>"ISO 639-1","Language name (in English)"
"AF","Afrikaans"
"AM","Amharic"
"AR","Arabic"
"AZ","Azerbaijani"
"BG","Bulgarian"
"BN","Bengali"
"BO","Tibetan"
"BS","Bosnian"
"CA","Catalan"
"CE","Chechen"
"CS","Czech"
"CY","Welsh"
"DA","Danish"
"DE","German"
"EL","Greek"
"EN","English"
"EO","Esperanto"
"ES","Spanish"
"ET","Estonian"
"EU","Basque"
"FA","Persian"
"FI","Finnish"
"FJ","Fijian"
"FR","French"
"HE","Hebrew"
"HI","Hindi"
"HR","Croatian"
"HT","Haitian Creole"
"HU","Hungarian"
"HY","Armenian"
"ID","Indonesian"
"IS","Icelandic"
"IT","Italian"
"JA","Japanese"
"JV","Javanese"
"KA","Georgian"
"KM","Khmer"
"KO","Korean"
"KS","Kashmiri"
"KU","Kurdish"
"LB","Luxembourgish"
"LI","Limburgish"
"LT","Lithuanian"
"LV","Latvian"
"MG","Malagasy"
"MI","Maori"
"MK","Macedonian"
"ML","Malayalam"
"MN","Mongolian"
"MO","Moldavian"
"MS","Malay"
"MT","Maltese"
"MY","Burmese"
"NA","Nauru"
"NE","Nepali"
"NG","Ndonga"
"NL","Dutch"
"NO","Norwegian"
"PL","Polish"
"PT","Portuguese"
"RO","Romanian"
"RU","Russian"
"SA","Sanskrit"
"SL","Slovenian"
"SM","Samoan"
"SO","Somali"
"SQ","Albanian"
"SR","Serbian"
"SS","Swati"
"ST","Sotho, Southern"
"SU","Sundanese"
"SV","Swedish"
"SW","Swahili"
"TA","Tamil"
"TH","Thai"
"TK","Turkmen"
"TL","Tagalog"
"TO","Tonga"
"TR","Turkish"
"TS","Tsonga"
"TY","Tahitian"
"UK","Ukrainian"
"UR","Urdu"
"UZ","Uzbek"
"VI","Vietnamese"
"XH","Xhosa"
"YI","Yiddish"
"ZH","Chinese"
"ZU","Zulu"<commit_msg>Sorted languages.csv file by language name, changed "Tagalog" to "Filipino"<commit_after>ISO 639-1,Language name (in English)
AF,Afrikaans
SQ,Albanian
AM,Amharic
AR,Arabic
HY,Armenian
AZ,Azerbaijani
EU,Basque
BN,Bengali
BS,Bosnian
BG,Bulgarian
MY,Burmese
CA,Catalan
CE,Chechen
ZH,Chinese
HR,Croatian
CS,Czech
DA,Danish
NL,Dutch
EN,English
EO,Esperanto
ET,Estonian
FJ,Fijian
TL,Filipino
FI,Finnish
FR,French
KA,Georgian
DE,German
EL,Greek
HT,Haitian Creole
HE,Hebrew
HI,Hindi
HU,Hungarian
IS,Icelandic
ID,Indonesian
IT,Italian
JA,Japanese
JV,Javanese
KS,Kashmiri
KM,Khmer
KO,Korean
KU,Kurdish
LV,Latvian
LI,Limburgish
LT,Lithuanian
LB,Luxembourgish
MK,Macedonian
MG,Malagasy
MS,Malay
ML,Malayalam
MT,Maltese
MI,Maori
MO,Moldavian
MN,Mongolian
NA,Nauru
NG,Ndonga
NE,Nepali
NO,Norwegian
FA,Persian
PL,Polish
PT,Portuguese
RO,Romanian
RU,Russian
SM,Samoan
SA,Sanskrit
SR,Serbian
SL,Slovenian
SO,Somali
ST,"Sotho, Southern"
ES,Spanish
SU,Sundanese
SW,Swahili
SS,Swati
SV,Swedish
TY,Tahitian
TA,Tamil
TH,Thai
BO,Tibetan
TO,Tonga
TS,Tsonga
TR,Turkish
TK,Turkmen
UK,Ukrainian
UR,Urdu
UZ,Uzbek
VI,Vietnamese
CY,Welsh
XH,Xhosa
YI,Yiddish
ZU,Zulu
{'fraction_non_alphanumeric': 0.21910331384015594, 'fraction_numerical': 0.0031189083820662767, 'mean_word_length': 11.64039408866995, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4613760', 'n_tokens_mistral': 1355, 'n_tokens_neox': 1246, 'n_words': 200}