<commit_before>b6h
c6h
d6h
f6h
g6h
m6h
p6h
s6h
t6h
a5h
e5h
i5h
o5h
u5h
5h
5h
5h
5h
5h
l5h
n5h
r5h<commit_msg>Final, failed version of this.<commit_after>a5h
5h
a6e
a6
a6o
6s
b6h
b6u
c6h
c6o
d6h
e5h
5h
e6o
f6h
f6o
f6u
g6h
i5h
5h
i6a
i6o
i6
i6u
i6
l5h
m6h
n5h
n6n
o5h
5h
o6
6r
p6h
r5h
r6r
s6h
t6h
u5h
5h
u6a
u6
u6b
u6c
u6g
u6i
u6
u6l
u6m
u6n
u6r
u6t
{'fraction_non_alphanumeric': 0.023054755043227664, 'fraction_numerical': 0.2132564841498559, 'mean_word_length': 3.4615384615384617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2912395', 'n_tokens_mistral': 298, 'n_tokens_neox': 297, 'n_words': 77}