<commit_before>Language name,Language code,Criterion ID
Arabic,ar,1019
Bengali,bn,1056
Brazilian Portuguese,pt_BR,1016
Bulgarian,bg,1020
Catalan,ca,1038
Chinese,zh,1008
Chinese (simplified),zh_CN,1017
Chinese (traditional),zh_TW,1018
Croatian,hr,1039
Czech,cs,1021
Danish,da,1009
Dutch,nl,1010
English,en,1000
English (US),en_US,1006
English (GB),en_GB,1007
Estonian,et,1043
Filipino,tl,1042
Finnish,fi,1011
French,fr,1002
German,de,1001
Greek,el,1022
Hebrew,iw,1027
Hindi,hi,1023
Hungarian,hu,1024
Icelandic,is,1026
Indonesian,id,1025
Italian,it,1004
Japanese,ja,1005
Korean,ko,1012
Latvian,lv,1028
Lithuanian,lt,1029
Malay,ms,1102
Norwegian,no,1013
Persian,fa,1064
Polish,pl,1030
Portuguese,pt,1014
Romanian,ro,1032
Russian,ru,1031
Serbian,sr,1035
Slovak,sk,1033
Slovenian,sl,1034
Spanish,es,1003
Swedish,sv,1015
Tamil,ta,1130
Telugu,te,1131
Thai,th,1044
Turkish,tr,1037
Ukrainian,uk,1036
Urdu,ur,1041
Vietnamese,vi,1040
N/A,1045,aa
N/A,1046,ab
N/A,1047,af
N/A,1048,am
N/A,1049,as
N/A,1050,ay
N/A,1051,az
N/A,1052,ba
N/A,1053,be
N/A,1054,bh
N/A,1055,bi
N/A,1056,bn
N/A,1057,bo
N/A,1058,br
N/A,1059,co
N/A,1060,cy
N/A,1061,nn
N/A,1062,eo
N/A,1063,eu
N/A,1064,fa
N/A,1065,fj
N/A,1066,fo
N/A,1067,fy
N/A,1068,ga
N/A,1069,gd
N/A,1070,gl
N/A,1071,gn
N/A,1072,gu
N/A,1073,gv
N/A,1074,ha
N/A,1075,bs
N/A,1076,hy
N/A,1077,ia
N/A,1078,ie
N/A,1079,ik
N/A,1080,iu
N/A,1081,jv
N/A,1082,ka
N/A,1083,kk
N/A,1084,kl
N/A,1085,km
N/A,1086,kn
N/A,1087,ks
N/A,1088,ku
N/A,1089,kw
N/A,1090,ky
N/A,1091,la
N/A,1092,lb
N/A,1093,ln
N/A,1094,lo
N/A,1095,mg
N/A,1096,mi
N/A,1097,mk
N/A,1098,ml
N/A,1099,mn
N/A,1100,mo
N/A,1101,mr
N/A,1102,ms
N/A,1103,mt
N/A,1104,my
N/A,1105,na
N/A,1106,ne
N/A,1107,oc
N/A,1108,om
N/A,1109,or
N/A,1110,pa
N/A,1111,ps
N/A,1112,qu
N/A,1113,rm
N/A,1114,rn
N/A,1115,rw
N/A,1116,sa
N/A,1117,sd
N/A,1118,se
N/A,1119,sg
N/A,1120,zu
N/A,1121,si
N/A,1122,sm
N/A,1123,sn
N/A,1124,so
N/A,1125,sq
N/A,1126,ss
N/A,1127,st
N/A,1128,su
N/A,1129,sw
N/A,1130,ta
N/A,1131,te
N/A,1132,tg
N/A,1133,ti
N/A,1134,tk
N/A,1135,tn
N/A,1136,to
N/A,1137,ts
N/A,1138,tt
N/A,1139,tw
N/A,1140,ug
N/A,1141,uz
N/A,1142,vo
N/A,1143,wo
N/A,1144,xh
N/A,1145,yi
N/A,1146,yo
N/A,1147,za
N/A,1148,en_AU
N/A,1149,pt_PT
N/A,1150,zh_HK<commit_msg>Fixing languge codes format.<commit_after>Language name,Language code,Criterion ID
Arabic,ar,1019
Bengali,bn,1056
Brazilian Portuguese,pt_BR,1016
Bulgarian,bg,1020
Catalan,ca,1038
Chinese,zh,1008
Chinese (simplified),zh_CN,1017
Chinese (traditional),zh_TW,1018
Croatian,hr,1039
Czech,cs,1021
Danish,da,1009
Dutch,nl,1010
English,en,1000
English (US),en_US,1006
English (GB),en_GB,1007
Estonian,et,1043
Filipino,tl,1042
Finnish,fi,1011
French,fr,1002
German,de,1001
Greek,el,1022
Hebrew,iw,1027
Hindi,hi,1023
Hungarian,hu,1024
Icelandic,is,1026
Indonesian,id,1025
Italian,it,1004
Japanese,ja,1005
Korean,ko,1012
Latvian,lv,1028
Lithuanian,lt,1029
Malay,ms,1102
Norwegian,no,1013
Persian,fa,1064
Polish,pl,1030
Portuguese,pt,1014
Romanian,ro,1032
Russian,ru,1031
Serbian,sr,1035
Slovak,sk,1033
Slovenian,sl,1034
Spanish,es,1003
Swedish,sv,1015
Tamil,ta,1130
Telugu,te,1131
Thai,th,1044
Turkish,tr,1037
Ukrainian,uk,1036
Urdu,ur,1041
Vietnamese,vi,1040
N/A,aa,1045
N/A,ab,1046
N/A,af,1047
N/A,am,1048
N/A,as,1049
N/A,ay,1050
N/A,az,1051
N/A,ba,1052
N/A,be,1053
N/A,bh,1054
N/A,bi,1055
N/A,bn,1056
N/A,bo,1057
N/A,br,1058
N/A,co,1059
N/A,cy,1060
N/A,nn,1061
N/A,eo,1062
N/A,eu,1063
N/A,fa,1064
N/A,fj,1065
N/A,fo,1066
N/A,fy,1067
N/A,ga,1068
N/A,gd,1069
N/A,gl,1070
N/A,gn,1071
N/A,gu,1072
N/A,gv,1073
N/A,ha,1074
N/A,bs,1075
N/A,hy,1076
N/A,ia,1077
N/A,ie,1078
N/A,ik,1079
N/A,iu,1080
N/A,jv,1081
N/A,ka,1082
N/A,kk,1083
N/A,kl,1084
N/A,km,1085
N/A,kn,1086
N/A,ks,1087
N/A,ku,1088
N/A,kw,1089
N/A,ky,1090
N/A,la,1091
N/A,lb,1092
N/A,ln,1093
N/A,lo,1094
N/A,mg,1095
N/A,mi,1096
N/A,mk,1097
N/A,ml,1098
N/A,mn,1099
N/A,mo,1100
N/A,mr,1101
N/A,ms,1102
N/A,mt,1103
N/A,my,1104
N/A,na,1105
N/A,ne,1106
N/A,oc,1107
N/A,om,1108
N/A,or,1109
N/A,pa,1110
N/A,ps,1111
N/A,qu,1112
N/A,rm,1113
N/A,rn,1114
N/A,rw,1115
N/A,sa,1116
N/A,sd,1117
N/A,se,1118
N/A,sg,1119
N/A,zu,1120
N/A,si,1121
N/A,sm,1122
N/A,sn,1123
N/A,so,1124
N/A,sq,1125
N/A,ss,1126
N/A,st,1127
N/A,su,1128
N/A,sw,1129
N/A,ta,1130
N/A,te,1131
N/A,tg,1132
N/A,ti,1133
N/A,tk,1134
N/A,tn,1135
N/A,to,1136
N/A,ts,1137
N/A,tt,1138
N/A,tw,1139
N/A,ug,1140
N/A,uz,1141
N/A,vo,1142
N/A,wo,1143
N/A,xh,1144
N/A,yi,1145
N/A,yo,1146
N/A,za,1147
N/A,en_AU,1148
N/A,pt_PT,1149
N/A,zh_HK,1150
{'fraction_non_alphanumeric': 0.19415073115860518, 'fraction_numerical': 0.2807649043869516, 'mean_word_length': 7.157798165137614, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6923819', 'n_tokens_mistral': 3509, 'n_tokens_neox': 2772, 'n_words': 332}