<commit_before>var conformanceFiles = [
	"ff-conformance/thefiles/01_simple.mp4",
	"ff-conformance/thefiles/02_dref_edts_img.mp4",
	"ff-conformance/thefiles/03_hinted.mp4",
	"ff-conformance/thefiles/04_bifs_video.mp4",
	"ff-conformance/thefiles/05_bifs_video_protected.mp4",
	"ff-conformance/thefiles/06_bifs.mp4",
	"ff-conformance/thefiles/07_bifs_sprite.mp4",
	"ff-conformance/thefiles/08_bifs_carousel.mp4",
	"ff-conformance/thefiles/09_text.mp4",
	"ff-conformance/thefiles/10_fragments.mp4",
	"ff-conformance/thefiles/12_metas.mp4",
	"ff-conformance/thefiles/13_long.mp4",
	"ff-conformance/thefiles/a1-foreman-QCIF.mp4",
	"ff-conformance/thefiles/a10-foreman_QCIF-raw.mp4",
	"ff-conformance/thefiles/a2-foreman-QCIF-hinted.mp4",
	"ff-conformance/thefiles/a4-tone-fragmented.mp4",
	"ff-conformance/thefiles/a5-foreman-AVC.mp4",
	"ff-conformance/thefiles/a7-tone-oddities.mp4",
	"ff-conformance/thefiles/a8-foreman_QCIF_edit.mp4",
	"ff-conformance/thefiles/a9-aac-samplegroups-edit.mp4",
	"ff-conformance/thefiles/f1.mp4",
	"ff-conformance/thefiles/f2.mp4",
	"ff-conformance/thefiles/fragment-random-access-1+AF8-rev1.mp4",
	"ff-conformance/thefiles/fragment_random_access-2.mp4",
	"ff-conformance/thefiles/timed-metadata.mp4",
	"ff-conformance/thefiles/a3/a3b-tone-deprot.mp4",
	"ff-conformance/thefiles/a3/a3-tone-protected.mp4",
	"ff-conformance/thefiles/a6/a6_tone_multifile.mp4",
	"ff-conformance/thefiles/compact-no-code-fec-1.iso3",
	"ff-conformance/thefiles/compact-no-code-fec-2.iso3",
	"ff-conformance/thefiles/female_amr67_hinted.3gp",
	"ff-conformance/thefiles/female_amr67DTX_hinted.3gp",
	"ff-conformance/thefiles/male_amr122.3gp",
	"ff-conformance/thefiles/male_amr122DTX.3gp",
	"ff-conformance/thefiles/mbms-fec.iso3",
	"ff-conformance/thefiles/pdin_example.3gp",
	"ff-conformance/thefiles/rs_example.3gp",
	"ff-conformance/thefiles/sg-tl-st.mp4",
	"ff-conformance/thefiles/restricted.mp4",
	"ff-conformance/thefiles/alst_hvc1.mp4",
	"ff-conformance/thefiles/trgr_hvc1.mp4",
	"ff-conformance/thefiles/aggr_hvc1.mp4",
	"ff-conformance/thefiles/subs_slice_hvc1.mp4",
	"ff-conformance/thefiles/subs_tile_hvc1.mp4",
];<commit_msg>remove non conformant iso file from unit tests<commit_after>var conformanceFiles = [
	"ff-conformance/thefiles/01_simple.mp4",
	"ff-conformance/thefiles/02_dref_edts_img.mp4",
	"ff-conformance/thefiles/03_hinted.mp4",
	"ff-conformance/thefiles/04_bifs_video.mp4",
	"ff-conformance/thefiles/05_bifs_video_protected.mp4",
	"ff-conformance/thefiles/06_bifs.mp4",
	"ff-conformance/thefiles/07_bifs_sprite.mp4",
	"ff-conformance/thefiles/08_bifs_carousel.mp4",
	"ff-conformance/thefiles/09_text.mp4",
	"ff-conformance/thefiles/10_fragments.mp4",
	"ff-conformance/thefiles/12_metas.mp4",
	"ff-conformance/thefiles/13_long.mp4",
	"ff-conformance/thefiles/a1-foreman-QCIF.mp4",
	"ff-conformance/thefiles/a10-foreman_QCIF-raw.mp4",
	"ff-conformance/thefiles/a2-foreman-QCIF-hinted.mp4",
	"ff-conformance/thefiles/a4-tone-fragmented.mp4",
	"ff-conformance/thefiles/a5-foreman-AVC.mp4",
	"ff-conformance/thefiles/a7-tone-oddities.mp4",
	"ff-conformance/thefiles/a8-foreman_QCIF_edit.mp4",
	"ff-conformance/thefiles/a9-aac-samplegroups-edit.mp4",
	"ff-conformance/thefiles/f1.mp4",
	"ff-conformance/thefiles/f2.mp4",
	"ff-conformance/thefiles/fragment-random-access-1+AF8-rev1.mp4",
	"ff-conformance/thefiles/fragment_random_access-2.mp4",
	"ff-conformance/thefiles/timed-metadata.mp4",
	"ff-conformance/thefiles/a3/a3b-tone-deprot.mp4",
	"ff-conformance/thefiles/a3/a3-tone-protected.mp4",
	"ff-conformance/thefiles/a6/a6_tone_multifile.mp4",
	"ff-conformance/thefiles/compact-no-code-fec-1.iso3",
	"ff-conformance/thefiles/compact-no-code-fec-2.iso3",
	"ff-conformance/thefiles/female_amr67_hinted.3gp",
	"ff-conformance/thefiles/female_amr67DTX_hinted.3gp",
	"ff-conformance/thefiles/male_amr122.3gp",
	"ff-conformance/thefiles/male_amr122DTX.3gp",
	"ff-conformance/thefiles/mbms-fec.iso3",
	"ff-conformance/thefiles/pdin_example.3gp",
	//"ff-conformance/thefiles/rs_example.3gp",
	"ff-conformance/thefiles/sg-tl-st.mp4",
	"ff-conformance/thefiles/restricted.mp4",
	"ff-conformance/thefiles/alst_hvc1.mp4",
	"ff-conformance/thefiles/trgr_hvc1.mp4",
	"ff-conformance/thefiles/aggr_hvc1.mp4",
	"ff-conformance/thefiles/subs_slice_hvc1.mp4",
	"ff-conformance/thefiles/subs_tile_hvc1.mp4",
];
{'fraction_non_alphanumeric': 0.16562572287763128, 'fraction_numerical': 0.04904001850566736, 'mean_word_length': 5.885350318471337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4202356', 'n_tokens_mistral': 2023, 'n_tokens_neox': 1949, 'n_words': 99}