<jupyter_start><jupyter_text>line() or delete()copy()  // copy originalcopy()  // copy originaladd()   // add a new line change() // overwrite line<jupyter_code>filenames = {"test":  '../original_corpus/eng.testb',
             "dev" :  '../original_corpus/eng.testa',
             "train": '../original_corpus/eng.train'}
folds = ['test','dev','train']
dtypes_in = {'correct_line':str}

log = []
log.append("line, num_added")

files = {}
for fold,filename in filenames.items(): 
    print(fold,filename)
    with open(filename, 'r') as file:
        files[fold] = file.readlines()

filenames
#read dataframe from file
corrections = pd.read_csv("../corrected_labels/token_edits.csv",dtype = dtypes_in)
corrections = corrections[['fold','line_no','doc_offset','corpus_span','correct_span','correct_line']]
corrections.head()

 
def print_with_numbering(file, s, e):
        for i in range(s,e):
            print(i, file[i])
        
def show_lines(line, file, line_end = None):
    line_end = line +2 if (line_end== None) else line_end
    print_with_numbering(file,line-1,line_end+1 )

def get_corpus_line(line):
    return data[line]
    

# run this to reset count
curr_row = 0; 
fold = "dev"
# for now just display
# when this bool is set to true, replace line with given text
prompt_for_correct = True;

for _,row in (corrections.loc[curr_row:curr_row+2]).iterrows():
    print(row.to_list())
    show_lines(row.line_no, files[row.fold])
        
if(prompt_for_correct):
    print("enter correct input for first entry")
    inp = input()
    if(inp == 'delete_entry'):
        corrections.drop(index=curr_row,axis = 1);
    else: 
        inp = inp.replace("\\n","\n")
        corrections.at[curr_row,'correct_line'] = inp
        print("\n line gotten: \n")
        print(inp)

curr_row += 1 
# curr_row -=1
display(curr_row)
corrections.head(curr_row).tail()
with open("../corrected_labels/token_corrections.json",'w') as file:
    corrections.to_json(file)
print("printed to file")
with open("temp_dump.json", 'w') as file:
    corrections.to_json(file)
corrections<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15359010936757014, 'fraction_numerical': 0.0038040893961008085, 'mean_word_length': 3.8256880733944953, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '485721', 'license': 'Apache-2.0', 'n_tokens_mistral': 745, 'n_tokens_neox': 730, 'n_words': 157, 'path': 'archive_old_files/scripts/Hackish_token_relabler.ipynb', 'repo_name': 'frreiss/Identifying-Incorrect-Labels-In-CoNLL-2003', 'size': '2103'}