<jupyter_start><jupyter_text>Generate code bin<jupyter_code>!g++ scripts/generate_csv.cc -I /usr/local/include/root/ -L /usr/local/lib/root/ -lTree -lCore -lPhysics -lRIO -o scripts/generate_csv
!g++ scripts/skim.cc -I /usr/local/include/root/ -L /usr/local/lib/root/ -lTree -lCore -lPhysics -lRIO -o scripts/skim  
!mkdir datasets/skims
!mkdir datasets/prepared_hlt_track
!mkdir datasets/prepared_hlt_body<jupyter_output><empty_output><jupyter_text>Generate data<jupyter_code>%%time
with open("skim.log", "w") as l:
    threads = []
    for ch,val in Samples.iteritems():
        threads.append(subprocess.Popen(["scripts/skim", str(ch), str(val["mother"]), str(len(val["children"])),
                                         "datasets"],
                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE))

    for thread in threads:
        t,e = thread.communicate()
        l.write(t)
        l.write("\n")
        l.write(e)
        l.write("\n")

with open("generate_hlt1.log", "w") as l:
    threads = []
    for ch,val in Samples.iteritems():
        threads.append(subprocess.Popen(["scripts/generate_csv", str(ch), str(val["mother"]), str(len(val["children"])),
                                         "datasets"],
                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE))

    for thread in threads:
        t,e = thread.communicate()
        l.write(t)
        l.write("\n")
        l.write(e)
        l.write("\n")
! cat generate_hlt1.log
!mkdir models
!mkdir bbdt_run2/<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15124441608168473, 'fraction_numerical': 0.0019144862795149968, 'mean_word_length': 2.408695652173913, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '395959', 'license': 'Apache-2.0', 'n_tokens_mistral': 529, 'n_tokens_neox': 520, 'n_words': 93, 'path': '0_Preprocessing.ipynb', 'repo_name': 'tlikhomanenko/LHCb-topo-trigger', 'size': '1567'}