<jupyter_start><jupyter_text>NORMAL MODE<jupyter_code>for text_file in info_files:
    model_file = text_file.split('.')[0] + '.lm'
    test_file = text_file.split('.')[0] + '.ppl'
    for i in reversed(range(0, order)):
        script = [str(command), '-text', str(text_file), '-order', str(i+1), '-lm', str(model_file), '-kndiscount']
        p = subprocess.Popen(script, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        a = p.wait()
        if a == 0:
            # print(' '.join(script), end='\n\n')
            f = open(test_file, 'w')
            script = [str(command2), '-lm', str(model_file), '-ppl', str(test_path), '-debug', str(2), '>', str(test_file)]
            # print(' '.join(script))
            p = subprocess.Popen(script, stdout=f, stderr=subprocess.PIPE)
            a = p.wait()
            break<jupyter_output>/Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/files/srilm/bin/macosx/ngram -lm /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/info30s.lm -ppl /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/test.txt -debug 2 > /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/info30s.ppl
/Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/files/srilm/bin/macosx/ngram -lm /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/info60s.lm -ppl /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/test.txt -debug 2 > /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/info60s.ppl
/Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/files/srilm/bin/macosx/ngram -lm /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/info5m.lm -ppl /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/test.txt -debug 2 > /Users/pablomaciasmunoz/Dev/WS_TFG/S2T_Training/resources/audio/tmp/info5m.ppl
/Us[...]<jupyter_text>UNKNOWED MODE<jupyter_code>for text_file in info_files:
    model_file = text_file.split('.')[0] + '_unk.lm'
    test_file = text_file.split('.')[0] + '_unk.ppl'
    for i in reversed(range(0, order)):
        script = [str(command), '-text', str(text_file), '-order', str(i + 1), '-lm', str(model_file), '-kndiscount',
                  '-unk']
        p = subprocess.Popen(script, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        a = p.wait()
        if a == 0:
            # print(' '.join(script), end='\n\n')
            f = open(test_file, 'w')
            script = [str(command2), '-lm', str(model_file), '-ppl', str(test_path), '-debug', str(2), '-unk', '>',
                      str(test_file)]
            # print(' '.join(script))
            p = subprocess.Popen(script, stdout=f, stderr=subprocess.PIPE)
            a = p.wait()
            break<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17101449275362318, 'fraction_numerical': 0.014130434782608696, 'mean_word_length': 3.215267175572519, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '583812', 'license': 'MIT', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1086, 'n_words': 142, 'path': 'notebooks/results.ipynb', 'repo_name': 'pamamu/S2T_Training', 'size': '2760'}