<jupyter_start><jupyter_text>FCE<jupyter_code>cos_sim_wl_fce = cosine_similarity(wl_mean_cluster_vectors, fce_mean_cluster_vectors)
cos_sim_wl_fce.shape
fce_sim_clusters = []
for i in range(1, 1515):
    ind = cos_sim_wl_fce[i].argsort()[::-1]
    good_ind = cos_sim_wl_fce[i][ind] > 0.85
    fce_sim_clusters.extend(ind[good_ind])
len(fce_sim_clusters)
fce_most_sim_cluster = list(pd.Series(fce_sim_clusters).value_counts().index)
len(set(fce_sim_clusters))<jupyter_output><empty_output><jupyter_text>Nucle<jupyter_code>cos_sim_wl_nucle = cosine_similarity(wl_mean_cluster_vectors, nucle_mean_cluster_vectors)
cos_sim_wl_nucle.shape
nucle_sim_clusters = []
for i in range(1, 1515):
    ind = cos_sim_wl_nucle[i].argsort()[::-1]
    good_ind = cos_sim_wl_nucle[i][ind] > 0.85
    nucle_sim_clusters.extend(ind[good_ind])
len(nucle_sim_clusters)
len(set(nucle_sim_clusters))
nucle_most_sim_cluster = list(pd.Series(nucle_sim_clusters).value_counts().index)<jupyter_output><empty_output><jupyter_text>Lang8<jupyter_code>cos_sim_wl_lang8 = cosine_similarity(wl_mean_cluster_vectors, lang8_mean_cluster_vectors)
cos_sim_wl_lang8.shape
lang8_sim_clusters = []
for i in range(1, 1515):
    ind = cos_sim_wl_lang8[i].argsort()[::-1]
    good_ind = cos_sim_wl_lang8[i][ind] > 0.85
    lang8_sim_clusters.extend(ind[good_ind])
len(lang8_sim_clusters)
len(set(lang8_sim_clusters))
lang8_most_sim_cluster = list(pd.Series(lang8_sim_clusters).value_counts().index)
def generate_text_dicts(dump):
    df = pd.DataFrame({"text_src":dump['text_src'], 'text_tgt': dump['text_tgt'], 'clusters':dump['clusters']})
    cluster_text_src = df.groupby(['clusters'])['text_src'].apply(lambda grp: list(grp)).to_dict()
    cluster_text_tgt = df.groupby(['clusters'])['text_tgt'].apply(lambda grp: list(grp)).to_dict()
    return cluster_text_src, cluster_text_tgt
fce_cluster_text_src, fce_cluster_text_tgt = generate_text_dicts(dump_fce)
nucle_cluster_text_src, nucle_cluster_text_tgt = generate_text_dicts(dump_nucle)
lang8_cluster_text_src, lang8_cluster_text_tgt = generate_text_dicts(dump_lang8)
fce_src = []
fce_tgt = []
for cl in fce_most_sim_cluster:
    fce_src.extend(fce_cluster_text_src.get(cl, []))
    fce_tgt.extend(fce_cluster_text_tgt.get(cl, []))
len(fce_src)
nucle_src = []
nucle_tgt = []
for cl in nucle_most_sim_cluster:
    nucle_src.extend(nucle_cluster_text_src.get(cl, []))
    nucle_tgt.extend(nucle_cluster_text_tgt.get(cl, []))
len(nucle_src)
lang8_src = []
lang8_tgt = []
for cl in lang8_most_sim_cluster:
    lang8_src.extend(lang8_cluster_text_src.get(cl, []))
    lang8_tgt.extend(lang8_cluster_text_tgt.get(cl, []))
len(lang8_src)
total_len = len(nucle_tgt)+len(fce_src)+len(lang8_src)
fce_prop = round(len(fce_src)/total_len,2)
nucle_prop = round(len(nucle_src)/total_len,2)
lang8_prop = round(len(lang8_src)/total_len,2)

print(100*fce_prop,"%")
print(100*nucle_prop,"%")
print(100*lang8_prop,"%")
c = 20000
fce_get = c*fce_prop
nucle_get = c*nucle_prop
lang8_get = c*lang8_prop
print(fce_get)
print(nucle_get)
print(lang8_get)
np.random.seed(4)
fce_ind = np.random.choice(np.arange(len(fce_src)), int(fce_get), replace=False)
nucle_ind = np.random.choice(np.arange(len(nucle_src)), int(nucle_get), replace=False)
lang8_ind = np.random.choice(np.arange(len(lang8_src)), int(lang8_get), replace=False)
final_scr = []
final_scr.extend([lang8_src[i] for i in lang8_ind])
final_scr.extend([nucle_src[i] for i in nucle_ind])
final_scr.extend([fce_src[i] for i in fce_ind])
final_scr.extend(dump_wl['text_src'])

len(final_scr)
final_tgt = []
final_tgt.extend([lang8_tgt[i] for i in lang8_ind])
final_tgt.extend([nucle_tgt[i] for i in nucle_ind])
final_tgt.extend([fce_tgt[i] for i in fce_ind])
final_tgt.extend(dump_wl['text_tgt'])

len(final_tgt)
from sklearn.model_selection import train_test_split
train_tgt, dev_tgt, train_src, dev_src = train_test_split(final_tgt, final_scr, test_size=0.02, random_state=4)
path = "../../data_parallel/clustering/"
# write_lines(path + "cluster_5k_train_src", train_src, mode='w')
# write_lines(path + "cluster_5k_train_tgt", train_tgt, mode='w')
# write_lines(path + "cluster_5k_dev_src", dev_src, mode='w')
# write_lines(path + "cluster_5k_dev_tgt", dev_tgt, mode='w')
# write_lines(path + "cluster_10k_train_src", train_src, mode='w')
# write_lines(path + "cluster_10k_train_tgt", train_tgt, mode='w')
# write_lines(path + "cluster_10k_dev_src", dev_src, mode='w')
# write_lines(path + "cluster_10k_dev_tgt", dev_tgt, mode='w')
# write_lines(path + "cluster_15k_train_src", train_src, mode='w')
# write_lines(path + "cluster_15k_train_tgt", train_tgt, mode='w')
# write_lines(path + "cluster_15k_dev_src", dev_src, mode='w')
# write_lines(path + "cluster_15k_dev_tgt", dev_tgt, mode='w')
write_lines(path + "cluster_20k_train_src", train_src, mode='w')
write_lines(path + "cluster_20k_train_tgt", train_tgt, mode='w')
write_lines(path + "cluster_20k_dev_src", dev_src, mode='w')
write_lines(path + "cluster_20k_dev_tgt", dev_tgt, mode='w')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1439107036077337, 'fraction_numerical': 0.022722742674905323, 'mean_word_length': 8.936633663366337, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '259739', 'license': 'Apache-2.0', 'n_tokens_mistral': 2213, 'n_tokens_neox': 2109, 'n_words': 261, 'path': 'notebooks/Clustering/Similarity between clusters.ipynb', 'repo_name': 'MaksTarnavskyi/improve-sequence-tagging', 'size': '5017'}