<commit_before>import os
import shutil
import unittest
from glob import glob

from omrdatasettools.downloaders.CapitanDatasetDownloader import CapitanDatasetDownloader
from omrdatasettools.image_generators.CapitanImageGenerator import CapitanImageGenerator


class CapitanImageGeneratorTest(unittest.TestCase):
    def test_download_extract_and_draw_bitmaps(self):
        # Arrange
        dataset_downloader = CapitanDatasetDownloader()
        image_generator = CapitanImageGenerator()

        # Act
        dataset_downloader.download_and_extract_dataset("temp/capitan_raw")
        symbols = image_generator.load_capitan_symbols("temp/capitan_raw")
        image_generator.draw_capitan_stroke_images(symbols, "temp/capitan_stroke", [3])
        image_generator.draw_capitan_score_images(symbols, "temp/capitan_score")

        # Assert
        all_stroke_images = [y for x in os.walk("temp/capitan_stroke") for y in glob(os.path.join(x[0], '*.png'))]
        all_score_images = [y for x in os.walk("temp/capitan_score") for y in glob(os.path.join(x[0], '*.png'))]
        self.assertEqual(10230, len(all_stroke_images))
        self.assertEqual(10230, len(all_score_images))

        # Cleanup
        os.remove(dataset_downloader.get_dataset_filename())
        shutil.rmtree("temp")


if __name__ == '__main__':
    unittest.main()<commit_msg>Greatly simplified and accelerated CapitanImageGeneratorTest<commit_after>import os
import shutil
import unittest
from glob import glob

from omrdatasettools.downloaders.CapitanDatasetDownloader import CapitanDatasetDownloader
from omrdatasettools.image_generators.CapitanImageGenerator import CapitanImageGenerator


test_data = r"""c_clef:0.0000,0.0000;0.0000,1.0619;-0.1180,2.7137;-0.3540,4.1295;-0.6490,5.9582;-0.7669,8.4949;-0.8849,13.8633;-0.9439,18.0518;-1.0619,22.0043;-1.1799,25.7208;-1.2979,29.2604;-1.4749,34.8647;-1.5339,38.3452;-1.5339,41.5898;-1.5339,44.3625;-1.5339,47.6661;-1.5339,49.9078;-1.5339,52.3265;-1.4159,54.6862;-1.2979,56.8690;-1.0619,60.1136;-0.8259,62.0013;-0.5900,63.7711;-0.2950,65.8358;0.0000,69.5524;0.1180,72.4430;0.3539,74.9797;0.4719,77.2804;0.7079,79.8171;1.1209,83.2977;1.3568,85.5984;1.4749,88.2531;1.5928,91.2617;1.7108,94.2113;1.8288,97.6919;1.9468,99.4617;1.9468,100.7595;2.0648,102.1754;2.1828,104.5351;2.3008,105.3020;2.3008,106.1869;3.5986,106.3048;3.6576,102.7653;3.6576,102.7653;14.9845,-5.4863;14.7486,-4.9554;14.5126,-3.4806;14.3946,-1.1209;14.3946,2.5957;14.5126,10.4417;14.8075,16.8719;15.0435,23.4791;15.0435,29.9093;15.0435,35.8086;15.0435,43.9496;15.1615,48.5510;15.1615,52.7395;15.1615,57.1049;15.1615,63.4171;15.1615,67.0747;15.1615,70.5553;15.2795,73.9178;15.5155,77.0445;15.8695,80.5840;16.1054,82.7078;16.2234,84.8315;16.3414,87.1322;16.4594,90.9077;16.5774,93.0905;16.8134,94.8603;16.9314,96.3941;17.0493,97.9279;17.2853,100.7595;17.4033,102.6473;17.4033,104.0041;17.4033,104.5941;17.4033,105.3020;16.9314,104.5351;16.5774,103.4732;16.5774,103.4732;59.0533,-13.1554;59.0533,-13.1554;58.4634,-13.0374;58.4634,-13.0374;58.2274,-12.1525;57.9324,-10.0288;57.8734,-7.9640;57.8734,-4.6014;57.8734,-0.4720;57.8734,7.4331;57.9914,13.8043;57.9914,20.2345;57.9914,26.4287;57.8734,32.2100;57.8734,40.2920;57.8734,45.2474;57.8734,49.6718;57.7554,53.6244;57.5195,57.3999;57.4015,62.9452;57.4015,66.5438;57.3425,70.0833;57.2245,73.3279;56.7525,77.8114;56.3986,81.1150;55.9856,84.2416;55.3367,87.2502;54.7467,89.9049;53.9208,94.3293;53.5078,97.9279;53.1539,101.1725;52.7409,104.3581;52.2690,108.1336;52.0920,109.6084;52.0330,110.1394;51.9740,110.7293;51.3841,108.8415;51.1481,104.7710;51.1481,104.7710;16.4594,15.6331;15.5745,16.1640;15.5745,16.1640;14.9845,17.5798;15.1025,18.7597;16.1054,20.2345;17.7573,21.8273;20.2350,23.3611;25.5445,25.4258;29.3202,26.9007;33.0368,28.3165;36.4585,29.6733;39.5852,30.2632;44.1867,30.7352;46.7235,31.3841;48.9653,32.0920;50.7941,32.7409;53.0359,33.6258;53.6258,33.8028;54.5107,33.8028;55.9266,33.9798;56.2806,33.2719;56.2806,33.2719;16.4004,34.2747;16.4004,34.2747;17.0493,34.8647;17.5803,35.4546;19.2321,36.9294;21.2379,38.1093;25.1316,40.1740;28.0223,41.8258;31.2080,43.3006;34.6886,44.1265;39.3492,45.0704;42.1809,45.6603;44.7177,46.2503;46.8415,46.4862;48.7293,46.4862;50.8531,46.3093;51.6790,46.1913;51.6790,46.1913;52.2690,46.3093;51.3841,46.0733;52.5049,47.3711;52.5049,47.3711;19.2321,53.3294;20.5300,54.2733;22.2408,55.4531;24.4236,56.6920;28.0813,58.5797;30.5591,59.4056;33.0958,60.1136;35.6326,60.4675;37.9923,61.1754;41.4140,62.4143;43.4198,63.0632;44.8947,63.0632;46.2516,63.1812;47.6674,63.1222;50.0272,62.8272;51.7380,62.7682;53.2129,63.2402;53.2129,63.2402;20.6480,70.5553;20.5890,71.0862;20.5890,71.0862;20.8840,71.6761;21.8869,72.5020;23.6567,73.2099;25.3086,73.8589;27.3734,74.6258;30.0281,75.8646;33.0368,76.5725;37.8743,76.6905;41.0600,76.5725;44.1277,76.1596;46.4285,75.8056;49.2013,75.5696;50.3811,75.5106;50.4991,74.6258;50.5581,72.6200;51.9150,72.5610;51.9150,72.5610:192,179,174,148,148,171,163,139,136,138,153,124,148,138,124,122,123,116,118,127,123,110,136,138,111,92,73,94,85,65,174,174,172,115,78,55,63,56,71,49,37,39,31,54,68,75,56,62,62,76,77,55,17,5,7,20,56,52,47,53,188,180,180,185,171,160,160,163,175,194,187,186,181,168,164,180,189,171,191,197,188,207,185,129,22,43,149,178,183,191,186,190,193,190,191,181,183,162,118,133,180,179,184,174,185,189,179,190,187,184,182,166,190,164,50,27,142,183,186,187,192,184,170,166,166,178,179,161,101,85,133,180,179,185,185,196,189,185,184,191,195,179,188,175,56,27,99,171,193,186,170,142,84,88,147,185,186,168,129,100,134,192,190,187,194,195,193,193,192,189,191,196,195,172,48,31,121,170,186,195,186,163,95,64,113,194,188,178,136,74,115,171,181,180,185,189,191,192,191,189,184,190,190,190,80,38,130,181,194,191,182,166,116,43,93,157,177,180,140,56,98,158,177,173,169,164,177,192,187,185,174,173,174,170,69,15,68,148,162,165,176,161,116,50,89,104,92,105,93,39,23,57,50,62,62,80,73,59,92,76,91,80,53,49,21,19,17,26,49,58,193,190,135,69,74,158,148,122,89,22,24,70,83,99,102,98,123,118,118,134,143,149,146,135,26,2,96,172,182,182,194,187,161,63,83,155,184,176,157,55,45,80,63,34,34,37,27,32,44,59,80,101,166,180,71,10,103,169,180,179,178,174,157,61,55,171,176,182,154,80,67,168,180,176,188,187,173,170,140,125,91,43,8,22,39,30,143,177,191,181,184,181,164,69,58,150,176,186,177,90,30,103,133,161,171,182,210,195,192,187,173,174,153,110,23,9,116,188,183,191,187,195,165,65,68,162,172,175,176,95,53,52,57,35,20,8,16,35,64,84,104,127,158,151,65,22,146,177,184,185,177,185,160,78,56,155,176,183,171,119,40,93,168,185,179,180,185,184,173,148,109,66,26,17,4,4,67,126,174,162,176,177,134,52,42,170,143,82,72,33,11,9,8,16,25,15,12,21,13,15,11,37,30,9,0,2,2,5,19,18,183,194,181,62,48,156,177,152,135,62,8,46,100,106,102,100,108,88,89,109,109,98,87,90,17,6,74,117,141,130,189,183,172,60,41,141,174,180,189,140,77,40,105,62,37,23,25,42,74,87,101,116,176,162,59,17,134,182,199,181,191,187,188,83,42,141,186,180,190,143,74,40,119,190,192,180,164,152,137,126,103,66,18,8,13,25,131,194,198,182,190,189,192,77,57,117,174,194,192,162,89,47,122,190,195,202,187,203,196,192,200,200,199,117,17,13,149,185,185,184,180,182,193,94,62,143,180,179,187,189,106,52,96,155,125,78,44,38,45,44,45,51,46,39,30,24,141,176,197,197,193,193,180,80,63,161,182,185,190,178,112,74,112,195,188,186,194,199,182,181,177,185,180,137,12,2,160,179,181,194,190,193,207,98,53,135,180,196,189,174,138,61,114,169,185,196,198,197,188,192,193,192,187,169,26,17,152,198,196,197,195,192,185,123,79,119,155,177,182,171,145,49,100,187,190,188,191,189,192,192,193,195,193,172,75,32,147,186,189,191,191,182,176,116,75,115,170,188,187,193,139,50,104,182,194,203,193,193,196,195,198,205,190,168,79,31,144,178,194,195,196,199,205,140,76,88,157,176,185,193,138,66,126,187,194,196,190,182,175,182,193,193,197,172,57,21,154,185,205,202,197,195,199,145,84,97,143,193,190,197,147,66,107,177,183,186,188,197,202,209,193,189,198,162,69,49,152,180,189,195,191,190,190,180,133,48,87,196,191,173,150,57,79,185,193,188,190,191,185,184,180,187,190,170,41,54,163,179,199,199,196,202,200,199,187,167,173,195,200,187,171,78,81,159,181,190,188,193,195,189,189,203,199,192,181,184,177,204,181,181,193,194,196,196,200,200,201,192,190,196,203,197,201,199,192,193,191,196,201,200,203,203,189,194,187,191,201,188,193,198
flat:0.0000,0.0000;0.0000,0.0000;0.0000,0.0000;-0.1770,1.1799;-0.2360,3.1266;-0.2360,7.9640;-0.1180,11.8575;0.0000,16.6360;0.0000,21.7683;-0.1180,30.7942;-0.4129,36.0446;-0.5899,40.9410;-0.7079,44.7755;-0.8259,48.0790;-0.9439,53.3294;-1.2389,56.3970;-1.5339,59.0517;-1.7108,60.9984;-1.7698,62.1783;-1.7698,62.8272;-1.7698,62.8272;-1.6518,61.5884;-1.2979,59.2877;-0.4719,54.6863;-0.2360,51.8546;0.1180,48.9049;0.7079,46.6043;1.4749,43.3006;1.9468,42.0028;2.5958,41.4129;3.0677,40.7639;3.7167,40.5870;4.7196,40.2920;6.8433,39.7611;8.4362,39.3481;9.7931,38.9942;11.0909,38.8762;12.3298,38.8762;13.2737,38.9942;13.9227,39.6431;14.2766,41.1769;14.3946,42.4747;14.3946,45.1884;14.1586,47.1352;13.8047,49.0819;13.0378,51.3826;11.7399,54.5682;10.5600,56.5150;9.2621,58.4618;7.8462,60.0546;6.0764,61.1754;3.3627,62.5323;1.5928,63.4171;-0.1180,64.3020;-1.7108,64.9510;-1.8288,66.0128;-1.4748,65.3639;-1.4748,65.3639:190,190,190,189,189,190,190,191,196,198,199,196,193,191,191,192,194,193,192,195,197,196,195,195,194,195,196,196,197,196,194,192,189,186,186,187,189,189,192,196,199,201,197,193,188,185,188,190,191,191,192,193,195,197,196,198,200,201,202,202,195,195,195,194,193,192,193,193,191,190,189,190,191,193,194,196,196,193,191,191,193,198,200,201,195,195,194,195,197,199,198,193,194,200,201,192,179,169,161,146,143,147,161,180,187,190,188,190,191,192,190,184,181,183,186,189,189,188,188,190,200,192,187,188,192,188,167,143,116,103,106,119,141,167,181,189,194,195,196,196,192,185,179,179,184,191,196,195,191,189,171,178,185,187,188,183,162,137,111,93,91,102,124,150,169,183,195,196,193,193,194,195,194,194,193,192,192,193,194,195,182,189,190,187,184,182,165,138,105,90,95,112,131,147,156,164,180,191,196,196,198,195,191,193,193,193,194,194,194,194,182,183,181,183,193,198,175,136,96,76,78,100,123,143,154,166,182,196,201,197,195,194,192,192,195,192,192,192,192,194,185,183,182,185,185,176,155,128,107,88,84,96,124,157,176,187,190,189,189,190,195,198,195,191,187,190,196,196,194,194,184,179,178,181,182,176,149,116,90,71,72,99,135,160,162,156,162,170,177,178,178,187,195,194,185,179,187,192,197,194,148,144,148,159,164,156,126,94,67,62,75,95,110,122,132,140,148,152,152,151,151,148,141,136,137,147,154,156,158,153,66,68,70,72,72,67,58,50,34,18,17,27,38,43,37,32,35,43,46,45,44,46,50,53,57,53,55,55,50,53,53,66,60,47,36,22,14,8,4,17,42,56,58,59,60,62,69,79,84,82,77,79,85,84,80,84,88,79,73,73,125,139,137,125,117,104,80,56,41,50,80,108,120,122,123,130,140,147,147,146,149,148,149,148,155,165,158,147,144,145,190,188,194,201,196,172,128,96,86,103,136,163,173,178,183,193,196,194,184,185,191,193,190,189,185,185,186,190,194,193,186,187,189,187,182,156,115,93,92,116,140,151,164,184,194,196,194,188,193,187,186,193,192,192,191,189,186,187,189,190,179,181,183,181,173,152,114,84,75,94,128,156,170,177,182,192,194,195,197,201,202,194,188,189,194,190,188,193,199,202,181,182,188,192,185,155,110,78,81,89,110,130,145,151,154,160,157,154,162,182,198,198,190,184,188,188,191,194,192,187,186,187,191,192,182,157,115,77,68,85,120,137,146,144,131,112,82,58,53,55,80,122,152,179,196,194,191,192,191,192,189,186,185,184,181,167,136,102,77,79,105,132,156,172,171,156,125,90,66,47,33,34,52,86,119,149,177,189,189,192,174,173,176,183,193,186,145,96,81,90,112,127,145,168,187,193,192,190,181,159,116,73,51,42,62,97,139,173,186,192,164,176,188,190,183,165,128,92,79,73,86,113,135,149,157,160,173,190,193,168,125,88,66,57,69,96,122,140,147,156,133,139,139,133,126,114,98,88,67,51,51,71,88,97,104,109,116,122,124,114,90,64,42,31,41,59,70,72,67,71,45,34,24,27,38,37,26,19,14,10,13,18,19,19,21,22,27,31,28,17,9,7,8,9,12,12,11,15,21,32,74,79,77,63,51,51,41,23,7,1,2,8,28,54,62,66,56,37,26,21,27,39,52,68,70,65,61,62,66,69,133,129,130,129,124,120,105,85,57,26,15,25,52,89,114,127,115,85,60,55,76,113,144,158,157,149,146,153,160,162,185,182,184,186,185,185,177,161,140,106,67,38,31,54,84,96,97,95,101,122,145,159,169,174,173,176,184,194,198,197,187,188,192,190,188,197,204,202,191,163,127,93,66,54,63,81,111,141,159,168,172,168,171,180,186,186,185,182,183,186,189,187,187,188,189,195,198,195,195,195,195,191,177,163,153,151,159,173,184,191,191,186,188,191,188,185,183,183,187,187,184,189,191,188,187,190,194,196,196,192,185,179,179,183,188,189,191,188,188,184,182,186,190,195,193,192,191,192,193,191
flat:0.0000,0.0000;-0.1770,1.9468;-0.3540,4.3065;-0.5899,8.6130;-0.8849,12.2115;-1.2389,15.8691;-1.9468,19.2906;-2.5367,24.7770;-3.1267,28.3165;-3.5397,31.6791;-3.9526,34.9237;-4.3066,37.9913;-4.6016,42.2978;-4.7785,44.7165;-4.8965,47.0172;-4.8965,49.2589;-4.7785,52.3266;-4.5426,53.9783;-4.4246,55.3941;-4.4246,56.9869;-4.4246,58.5208;-4.4246,60.4085;-4.4246,61.5294;-4.4246,62.5323;-4.5426,63.4172;-4.6606,64.2430;-4.6606,64.7740;-4.6606,64.7740;-4.6606,64.2430;-4.4246,62.8862;-4.1886,60.7625;-3.9526,58.8747;-3.6577,57.1640;-3.3627,55.6301;-3.0087,54.4503;-2.1828,53.6244;-0.8259,52.4445;0.2360,51.4417;1.0619,50.9697;1.6519,50.9697;3.7756,50.7927;4.7785,50.7927;5.9584,50.7927;7.1383,51.0877;8.6722,51.7956;9.2031,52.3855;10.1470,53.2114;11.1499,54.9222;11.9759,56.3970;12.6248,58.5797;13.0967,59.9366;13.0967,61.2344;12.8018,62.4143;11.7989,64.0071;10.9730,64.8920;10.0880,65.7179;8.9671,66.7207;7.6103,68.1366;5.7814,69.7884;5.0735,70.3783;4.3066,70.3193;3.3037,69.8474;2.3598,69.2574;1.3569,69.0215;0.5899,68.4905;-0.1180,68.1956;-1.4749,67.5466;-2.2418,66.4258;-2.5958,64.8330;-1.1209,64.7740;-1.1209,64.7740:172,187,184,184,184,186,188,187,190,189,187,188,191,191,190,190,192,193,194,195,193,193,196,200,204,206,201,196,193,191,171,187,190,193,191,187,185,184,184,188,189,187,187,186,184,181,185,191,198,204,204,201,202,204,209,212,209,203,199,195,168,181,181,187,187,183,180,178,180,183,185,183,181,177,175,179,180,179,178,181,185,186,187,189,192,194,194,193,191,189,170,176,181,185,192,196,191,183,180,181,166,136,115,106,111,156,183,196,197,197,188,181,178,180,185,193,194,192,194,197,179,181,185,192,196,193,193,199,201,197,167,116,83,83,107,167,187,197,193,188,183,186,187,191,191,192,190,192,195,196,181,195,200,197,197,191,184,184,182,170,143,109,89,87,104,147,161,172,180,185,190,194,193,189,186,184,182,176,172,174,168,180,180,186,183,180,177,182,177,161,141,111,92,83,101,144,160,168,173,174,183,187,193,191,187,183,177,168,160,156,76,96,107,110,115,123,124,119,105,86,69,59,62,62,63,67,73,88,98,91,88,100,116,116,107,99,98,104,106,101,36,37,36,32,32,36,48,50,41,23,15,13,21,34,54,61,51,42,39,54,60,56,46,44,50,58,67,77,82,75,153,157,151,144,136,132,127,127,113,80,50,40,58,82,115,150,149,143,138,139,139,137,137,132,130,132,140,150,161,172,170,175,173,178,188,196,200,190,156,110,71,65,93,117,143,181,187,199,211,213,207,197,190,194,198,199,193,185,182,184,169,176,177,178,180,179,191,184,149,104,69,67,96,118,139,173,182,188,192,197,199,193,191,187,189,189,187,186,188,189,183,176,184,184,182,178,172,162,133,92,68,70,99,134,161,166,173,181,185,190,191,190,192,198,201,201,196,190,186,181,170,175,180,191,194,186,183,154,114,77,57,57,80,120,158,181,179,175,174,179,183,183,184,185,181,177,177,182,189,191,189,192,181,181,193,190,176,147,115,84,64,67,100,130,153,176,183,186,184,185,187,188,190,190,190,189,187,188,190,193,183,182,182,186,188,184,163,148,122,85,50,60,111,147,171,175,181,185,187,188,189,187,187,184,187,187,190,189,188,191,177,171,173,183,190,186,167,145,117,89,68,72,102,133,162,201,199,195,189,187,184,185,187,185,182,183,188,192,193,193,152,178,190,194,188,172,165,143,114,85,74,96,132,160,170,187,190,187,185,190,198,201,200,196,192,191,187,188,193,189,92,122,137,145,151,152,135,100,66,44,46,70,96,113,125,151,146,128,111,111,117,121,124,120,111,101,98,105,118,125,21,32,42,49,48,40,30,19,13,8,13,15,12,10,8,5,4,5,12,20,16,15,23,32,27,15,15,16,22,25,25,26,20,22,36,45,44,17,3,4,16,29,36,33,30,29,27,13,5,2,4,3,6,5,11,37,59,65,59,67,119,129,136,141,141,127,102,60,26,24,45,84,126,148,154,135,124,121,111,70,33,13,5,4,26,65,107,142,149,150,178,181,184,187,185,173,151,105,59,55,89,140,171,173,176,184,189,190,185,176,134,74,29,10,42,101,146,174,183,185,185,189,190,203,206,191,164,111,61,38,57,118,168,182,190,190,186,181,180,165,126,77,46,53,102,149,163,177,182,186,194,194,189,200,198,181,152,109,79,59,49,81,136,162,171,184,194,203,198,167,113,68,49,74,120,160,175,186,200,206,180,169,173,177,186,185,180,145,92,57,41,69,119,158,189,196,196,193,173,131,85,59,58,90,138,162,175,183,190,193,183,183,187,187,184,184,182,152,98,68,38,41,75,133,173,186,184,176,156,118,75,61,69,111,153,170,175,180,186,189,165,168,175,190,196,189,184,152,109,70,44,28,35,62,103,153,149,123,92,82,93,121,151,175,187,183,181,184,191,193,187,182,190,198,202,200,194,182,148,103,63,31,17,22,25,49,81,112,125,139,168,192,200,201,198,196,205,208,208,199,180,185,187,180,183,192,202,204,195,176,147,124,120,120,112,130,154,168,173,177,184,194,199,201,195,193,194,197,197,191"""

class CapitanImageGeneratorTest(unittest.TestCase):
    def test_download_extract_and_draw_bitmaps(self):
        # Arrange
        shutil.rmtree("temp", ignore_errors=True)
        image_generator = CapitanImageGenerator()
        data_directory = "temp/capitan_raw"
        data_path = os.path.join(data_directory, "BimodalHandwrittenSymbols", "data")
        os.makedirs(os.path.join(data_directory, "BimodalHandwrittenSymbols"))
        with open(data_path, "w") as data_file:
            data_file.write(test_data)

        # Act
        symbols = image_generator.load_capitan_symbols(data_directory)
        image_generator.draw_capitan_stroke_images(symbols, "temp/capitan_stroke", [3])
        image_generator.draw_capitan_score_images(symbols, "temp/capitan_score")

        # Assert
        all_stroke_images = [y for x in os.walk("temp/capitan_stroke") for y in glob(os.path.join(x[0], '*.png'))]
        all_score_images = [y for x in os.walk("temp/capitan_score") for y in glob(os.path.join(x[0], '*.png'))]
        self.assertEqual(3, len(all_stroke_images))
        self.assertEqual(3, len(all_score_images))

        # Cleanup
        shutil.rmtree("temp", ignore_errors=True)


if __name__ == '__main__':
    unittest.main()
{'fraction_non_alphanumeric': 0.2420919905706924, 'fraction_numerical': 0.6132339235787512, 'mean_word_length': 12.777945619335348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7243369', 'n_tokens_mistral': 16207, 'n_tokens_neox': 9695, 'n_words': 160}