<jupyter_start><jupyter_text>Try except predicted encoded to predicted<jupyter_code>os.getcwd()
books = [f for f in os.listdir("data/") if "sample" not in f and "DS_Store" not in f]
books
encoded_suffix = "_post_edited_encoded"
decoded_suffix = "_post_edited"
for b in books:
    try:
        book_path = "data/" + b
        book_encoded_path = book_path + "/" + b +  encoded_suffix + "/"
        book_decoded_path = book_path + "/" + b + decoded_suffix + "/"

        mapping_folders = [f for f in os.listdir(book_encoded_path) if "DS_Store" not in f]
    #     print(mapping_folders)
        for mf in mapping_folders:
            e_files = [f for f in os.listdir(book_encoded_path + mf) if "DS_Store" not in f]
            
            # make folder in decoded:
            try:
                os.mkdir(book_decoded_path + mf + "/")
                print(book_decoded_path + mf + "/", made)
                
            else:
                pass
            
            
            for f in e_files:
                try:
                    content = decode(open(book_encoded_path + mf + "/" + f, "r", encoding = "utf8").read())
                    output_name = f.replace("predicted_encoded", "predicted")
                    output_file = open(book_decoded_path + mf + "/" + output_name, "w", encoding = "utf8" )
                    output_file.write(content)
                    output_file.close()

                except:
                    print(b, mf, f)
                    
    except:
        pass
#         break
#     break
   
os.listdir("data/co")<jupyter_output><empty_output><jupyter_text>Unk Token<jupyter_code>import os
import pandas as pd
import subprocess
# Basic alignment - Ossama's code
def alignFilesBasic(OneEncodePrefix, OneEncodeFolder, TwoEncodePrefix, TwoEncodeFolder, saveAlignmentAs, alignerLocation, results_prefix):

    OneName = OneEncodePrefix + ".txt"
    TwoName = TwoEncodePrefix + ".txt"

    #"python align_text.py -r ocr_tokenized.txt -c rafed_tokenized.txt -m basic -o sample/sample.ar"
    command = "python3 " + alignerLocation + " -s " + OneEncodeFolder + OneName + " -t " + TwoEncodeFolder + TwoName 
    command +=  " -m basic -o " + saveAlignmentAs + results_prefix

    print(command)
    p = subprocess.getstatusoutput(command)
    print(p)
os.chdir("..")
os.getcwd()
book_name = "columbia_aco000270"
alignerLocation = "code/ced_word_alignment/align_text.py"

OneEncode = "ocr_space_output_encoded_"
OneEncodeFolder = "data/columbia_aco000270/columbia_aco000270_raw_ocr_encoded/"

TwoEncode = "predicted_encoded_"
TwoEncodeFolder = "data/columbia_aco000270/columbia_aco000270_post_edited_encoded/columbia_aco000270_model_msa_70m_map_pure_emp2_pg25_top30/"
saveAlignmentAs = "data/" + book_name + "/" + book_name + "_unk/"
results = "unk_file_"
start = 9
end = 19 # INCLUSIVE

for i in range(start, end + 1):   
    OneEncodePrefix = OneEncode + str(i)
    TwoEncodePrefix = TwoEncode + str(i)
    results_prefix = results + str(i)
    alignFilesBasic(OneEncodePrefix, OneEncodeFolder, TwoEncodePrefix, TwoEncodeFolder, saveAlignmentAs, alignerLocation, results_prefix)
"data/columbia_aco000270/columbia_aco000270_unk/"<jupyter_output><empty_output><jupyter_text>Fix UNK Business<jupyter_code>#function to decode text
def decode(l):
	#     l = l.replace("+", "#")
    # there is an unexpected case in disambig results - #A #B#. 
    # we are unsure of whether to split or merge this case, and currently we are spliting
    l = l.replace("# #", "").replace("+ ", "").replace("</s>", "").replace("<s>", "")
    l = l.replace("# #", "").replace("+ ", "").replace("</s>", "").replace("<s>", "")
    l = l.replace("# ", " ").replace(" #", " ")
    return l
import os
encoded_folder = "data/columbia_aco000270/columbia_aco000270_post_edited_encoded/columbia_aco000270_model_openiti_70m_map_emp_space_pg_25_top_15_no_unk/"
encoded_prefix = "predicted_encoded_"
raw_folder = "data/columbia_aco000270/columbia_aco000270_raw_ocr_encoded/"
raw_prefix = "ocr_space_output_encoded_"

encoded_output_folder = encoded_folder[:-1] + "_fixed/"
encoded_output_prefix = "predicted_encoded_"
decoded_output_folder = "data/columbia_aco000270/columbia_aco000270_post_edited/" + encoded_folder.split("/")[-2] + "_fixed/"
decoded_output_prefix = "predicted_"
try:
    os.mkdir(encoded_output_folder)
    print(encoded_output_folder, "made")
    
except:
    pass

try:
    os.mkdir(decoded_output_folder)
    print(decoded_output_folder, "made")

except:
    pass
start_page = 9
end_page = 19 # INCLUSIVE
for i in range(start_page, end_page + 1):
    unk_lines = open(encoded_folder + encoded_prefix + str(i) + ".txt", "r", encoding = "utf8").readlines()
    raw_lines = open(raw_folder + raw_prefix + str(i) + ".txt", "r", encoding = "utf8").readlines()
    
    for j in range(len(unk_lines)):
        unk_line = unk_lines[j].split(" ")
        raw_line = raw_lines[j].split(" ")
        
        if len(unk_line) != len(raw_line):
            print("page", i, "line", j, "does not align")
            
        else:
            for k in range(len(unk_line)):
                if unk_line[k].strip() == "<unk>":
#                     print("replacing")
                    unk_line[k] = raw_line[k]
                    
            unk_lines[j] = " ".join(unk_line)
#             print(unk_lines[j])

    encoded_output = open(encoded_output_folder + encoded_output_prefix + str(i) + ".txt", "w", encoding = "utf8")
    encoded_output.write("".join(unk_lines))
    encoded_output.close()
    
    
    decoded_output = open(decoded_output_folder + decode_output_prefix + str(i) + ".txt", "w", encoding = "utf8")
    decoded_output.write(decode("".join(unk_lines)))
    decoded_output.close()
    
#     break
unk_lines<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12640498011412762, 'fraction_numerical': 0.020750475531730934, 'mean_word_length': 3.0362875087229586, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '78796', 'license': 'Apache-2.0', 'n_tokens_mistral': 2042, 'n_tokens_neox': 1909, 'n_words': 384, 'path': 'code/misc.ipynb', 'repo_name': 'CAMeL-Lab/arafix_ocr', 'size': '5783'}