<jupyter_start><jupyter_text>WL<jupyter_code>wl_src = read_lines("../../data_parallel/wi+locness/train_src")
wl_tgt = read_lines("../../data_parallel/wi+locness/train_tgt")<jupyter_output><empty_output><jupyter_text>FCE<jupyter_code>fce_texts_src = read_lines("../../data_parallel/fce/fce_train_src")
fce_texts_tgt = read_lines("../../data_parallel/fce/fce_train_tgt")
with open("fce_cos.pickle", "rb") as f:
    fce_cos = pickle.load(f)
arg_s = fce_cos.argsort()
def create_df(text_src,text_tgt, cos_sim, only_change=True):
    df = pd.DataFrame({"text_src": text_src, "text_tgt": text_tgt, "cos_sim":cos_sim})
    df = df.drop_duplicates(subset=['text_src','text_tgt'])
    if only_change:
        df = df[df["text_src"] != df["text_tgt"]]
    df = df.sort_values('cos_sim')
    return df
fce_df = create_df(fce_texts_src,fce_texts_tgt, fce_cos)
fce_df.head(50)
fce_df.tail(50)
# for i in arg_s[:800]:
#     print(fce_cos[i])
#     print(fce_texts_src[i])
#     print(fce_texts_tgt[i])
#     print()<jupyter_output><empty_output><jupyter_text>Nucle<jupyter_code>nucle_texts_src = read_lines("../../data_parallel/nucle/nucle_src")
nucle_texts_tgt = read_lines("../../data_parallel/nucle/nucle_tgt")
with open("nucle_cos.pickle", "rb") as f:
    nucle_cos = pickle.load(f)
df_nucle = pd.DataFrame({"text_src": nucle_texts_src, "text_tgt": nucle_texts_tgt, "cos_sim":nucle_cos})
df_nucle = df_nucle.drop_duplicates()
len(df_nucle)
len(df_nucle)
df_nucle_use = df_nucle[df_nucle["text_src"] != df_nucle["text_tgt"]].sort_values('cos_sim')
df_nucle_use
df_nucle_use.head(20)
df_nucle_use.reset_index(inplace=True)
df_nucle_use.loc[100:120]
df_nucle_use.tail(20)
nucle_df = create_df(nucle_texts_src,nucle_texts_tgt, nucle_cos)
nucle_df.tail(30)
# for i in arg_nucle[:500]:
#     print(nucle_cos[i])
#     print(nucle_texts_src[i])
#     print(nucle_texts_tgt[i])
#     print()
# for i in arg_nucle[-500:]:
#     print(nucle_cos[i])
#     print(nucle_texts_src[i])
#     print(nucle_texts_tgt[i])
#     print()<jupyter_output><empty_output><jupyter_text>Lang8<jupyter_code>lang8_texts_src = read_lines("../../data_parallel/lang8/lang8_src")
lang8_texts_tgt = read_lines("../../data_parallel/lang8/lang8_tgt")
with open("lang8_cos.pickle", "rb") as f:
    lang8_cos = pickle.load(f)
lang8_df = create_df(lang8_texts_src,lang8_texts_tgt, lang8_cos)
lang8_df.head(50)
lang8_df.tail(50)
fce_df.cos_sim.describe()
lang8_df.cos_sim.describe()
nucle_df.cos_sim.describe()
fce_df.reset_index(inplace=True)
nucle_df.reset_index(inplace=True)
lang8_df.reset_index(inplace=True)
fce_df = fce_df.loc[10000:]
nucle_df = nucle_df.loc[10000:]
lang8_df = lang8_df.loc[10000:]
nucle_df
lang8_df
final_df = lang8_df.head(2000)
final_df = final_df.append(nucle_df.head(1500))
final_df = final_df.append(fce_df.head(1500))
final_src = []
final_tgt = []
final_src.extend(final_df.text_src.values)
final_tgt.extend(final_df.text_tgt.values)
final_src.extend(wl_src)
final_tgt.extend(wl_tgt)
len(final_src)
from sklearn.model_selection import train_test_split
train_tgt, dev_tgt, train_src, dev_src = train_test_split(final_tgt, final_src, test_size=0.02, random_state=4)
path = "../../data_parallel/checkpoints/"
def write_lines(fn, lines, mode='w'):
    text_to_write = "\n".join(lines)
    with open(fn, encoding='utf-8', mode=mode) as f:
        f.write(text_to_write)
# write_lines(path + "check_5k_train_src", train_src, mode='w')
# write_lines(path + "check_5k_train_tgt", train_tgt, mode='w')
# write_lines(path + "check_5k_dev_src", dev_src, mode='w')
# write_lines(path + "check_5k_dev_tgt", dev_tgt, mode='w')
write_lines(path + "check_5k_train_src_v3", train_src, mode='w')
write_lines(path + "check_5k_train_tgt_v3", train_tgt, mode='w')
write_lines(path + "check_5k_dev_src_v3", dev_src, mode='w')
write_lines(path + "check_5k_dev_tgt_v3", dev_tgt, mode='w')
write_lines(path + "check_10k_train_src", train_src, mode='w')
write_lines(path + "check_10k_train_tgt", train_tgt, mode='w')
write_lines(path + "check_10k_dev_src", dev_src, mode='w')
write_lines(path + "check_10k_dev_tgt", dev_tgt, mode='w')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15853362466618112, 'fraction_numerical': 0.024763292061179897, 'mean_word_length': 7.031189083820663, 'pattern_counts': {'":': 6, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '33248', 'license': 'Apache-2.0', 'n_tokens_mistral': 1904, 'n_tokens_neox': 1776, 'n_words': 215, 'path': 'notebooks/Checkpoint_exp/Generete_dataset.ipynb', 'repo_name': 'MaksTarnavskyi/improve-sequence-tagging', 'size': '4119'}