<commit_before>random test
0 1 9000
1 2 181
2 3 -60214
3 4 86532
4 5 2163
5 6 -147150
6 7 55083
7 8 -253055
8 9 66206
9 10 160656
10 11 54440
11 12 36467
12 13 98798
13 14 -75236
14 15 167222
15 16 -208566
16 17 -55658
17 18 83119
18 19 160339
19 20 93497
20 21 129258
21 22 -191095
22 23 20865
23 24 -60256
24 25 149432
25 26 27747
26 27 113046
27 28 -108123
28 29 189281
29 30 76155
30 31 -47399
31 32 -27925
32 1 166379
33 34 60827
34 35 72934
35 36 -88973
36 37 113329
37 38 96198
38 39 317618
39 40 47606
40 41 -15365
41 42 31479
42 43 159372
43 44 -125665
44 45 13470
45 46 -189094
46 47 37572
47 48 79911
48 49 -151562
49 50 59333
50 51 -91923
51 52 47081
52 53 59976
53 54 -19296
54 55 -138414
55 56 -26894
56 57 112813
57 58 107342
58 59 -51393
59 60 264207
60 61 -47266
61 62 -1284
62 63 -26415
63 64 39808
64 33 18731
65 66 106828
66 67 -116670
67 68 -106254
68 69 -46928
69 70 42080
70 71 -117263
71 72 79361
72 73 -21809
73 74 -155238
74 75 86164
75 76 -170594
76 77 -90904
77 78 89880
78 79 -36055
79 80 335998
80 81 154599
81 82 126531
82 83 87669
83 84 50332
84 85 -55687
85 86 -46965
86 87 22889
87 88 23038
88 89 -52607
89 90 -74702
90 91 -70841
91 92 117841
92 93 198421
93 94 -15514
94 95 -119205
97 98 10167
98 99 230453
99 100 45537
101 96 100<commit_msg>identified evil line, see diff<commit_after>random test
0 1 9000
1 2 181
2 3 -60214
3 4 86532
4 5 2163
5 6 -147150
6 7 55083
7 8 -253055
8 9 66206
9 10 160656
10 11 54440
11 12 36467
12 13 98798
13 14 -75236
14 15 167222
15 16 -208566
16 17 -55658
17 18 83119
18 19 160339
19 20 93497
20 21 129258
21 22 -191095
22 23 20865
23 24 -60256
24 25 149432
25 26 27747
26 27 113046
27 28 -108123
28 29 189281
29 30 76155
30 31 -47399
31 32 -27925
32 1 166379
33 34 60827
34 35 72934
35 36 -88973
36 37 113329
37 38 96198
38 39 317618
39 40 47606
40 41 -15365
41 42 31479
42 43 159372
43 44 -125665
44 45 13470
45 46 -189094
46 47 37572
47 48 79911
48 49 -151562
49 50 59333
50 51 -91923
51 52 47081
52 53 59976
53 54 -19296
54 55 -138414
55 56 -26894
56 57 112813
57 58 107342
58 59 -51393
59 60 264207
60 61 -47266
61 62 -1284
62 63 -26415
63 64 39808
64 33 18731
65 66 106828
66 67 -116670
67 68 -106254
68 69 -46928
69 70 42080
70 71 -117263
71 72 79361
72 73 -21809
73 74 -155238
74 75 86164
75 76 -170594
76 77 -90904
77 78 89880
78 79 -36055
79 80 335998
80 81 154599
81 82 126531
82 83 87669
83 84 50332
84 85 -55687
85 86 -46965
86 87 22889
87 88 23038
88 89 -52607
89 90 -74702
90 91 -70841
91 92 117841
92 93 198421
93 94 -15514
94 95 -119205
96 65 -141304
97 98 10167
98 99 230453
99 100 45537
101 96 100
{'fraction_non_alphanumeric': 0.03404255319148936, 'fraction_numerical': 0.7017408123791102, 'mean_word_length': 2.7751824817518247, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4698815', 'n_tokens_mistral': 2438, 'n_tokens_neox': 1146, 'n_words': 604}