<jupyter_start><jupyter_text>Prepares the data from a ids file and a language<jupyter_code>def prepare_datasets(language='latvian'):
    ids_path = "../articles/"+language+"_ids"
    if language == 'estonian':
        lang = 'ee'
    else:
        lang = 'lv'
    ids_list = []
    with open(ids_path, "r") as f:
        for id_ in f:
            ids_list.append(int(id_))
    id2article = {}
    
    for years in ["2018.json", "2019.json"]:
        path = "../articles/" + lang + "_" + years
        articles = json.load(open(path, 'r', encoding='utf-8'))
        for art in articles:
            if int(art['id']) in ids_list:
                id2article[art['id']] = art['title'].lower() + art['bodyText'].lower()
    return id2article
def embedd_texts(text):
    from sentence_transformers import SentenceTransformer
    model = SentenceTransformer('xlm-r-distilroberta-base-paraphrase-v1')
    embedded = model.encode(text)
    return embedded    
latvian_articles = prepare_datasets('latvian')
embedded_lv = embedd_texts(list(latvian_articles.values()))
with open('latvan_embeddings.pkl', "wb") as f:
    pickle.dump(embedded_lv, f)<jupyter_output><empty_output><jupyter_text>Load proxy articles for Estonian "Lati delfi" documents<jupyter_code>delfi_to_ee = pickle.load(open("../estonian_to_latvian_sbert_rankings.pkl","rb"))
#Extract unique ids
unique_ids = set([x for c in delfi_to_ee for x in delfi_to_ee[c]])
reducer = umap.UMAP()
def project_data(data):
    scaled_data = StandardScaler().fit_transform(data)
    l1_norm = np.abs(scaled_data).sum(axis=1)
    scaled_data = scaled_data / l1_norm[:, np.newaxis]
    embedding = reducer.fit_transform(scaled_data)
    return embedding
#Read original 100 articles
original_delfi_data = json.load(open(filepaths['latvian'], 'r'))['list']
delfi_embed = embedd_texts(original_delfi_data)
ids2texts = pickle.load(open("../pickles/latvian_embeddings.pkl","rb"))
embedd_docs = pickle.load(open("../pickles/latvian_pickle.pkl","rb"))
#embedd_docs = embedd_texts(list(ids2texts.values()))
ids2embs = dict(zip(ids2texts.keys(), embedd_docs))
#
def project(delfi_embed, ids2embs, parameter = 10):
    project_ids = set([x for delf in delfi_to_ee for x in delfi_to_ee[delf][:parameter]])
    ee_outs = [ids2embs[key] for key in project_ids]
    merge_embs = np.vstack((ee_outs, delfi_embed))
    merge_proj = project_data(merge_embs)
    delf_len = len(delfi_embed) #.shape[0]
    main_len = len(ee_outs)#.shape[0]
    colors = ["blue"] * main_len + ['red'] * delf_len
    plt.xlabel('UMAP-1D')  
    plt.ylabel('UMAP-2D')  
    plt.title("Delfi to estonian @ " + str(parameter)) 
    plt.scatter(merge_proj[:, 0], merge_proj[:, 1], s = 10, color = colors)
    
import os
os.listdir("../../")

tarmos_list_f = open("../pairs_latvian_part.c","r", encoding='utf-8')
tarmos_files = [ line.strip() for line in tarmos_list_f.readlines() ]

embedd_tarmos = embedd_texts(tarmos_files)

with open('../pickles/latvian_embeddings.pkl', "rb") as f:
    ids2texts_lv = pickle.load(f)
embedd_lv = pickle.load(open("../pickles/latvian_pickle.pkl","rb"))
def project2(embedd_tarmos, delfi_embed, ids2embs, parameter = 10):
    project_ids = set([x for delf in delfi_to_ee for x in delfi_to_ee[delf][:parameter]])
    ee_outs = [ids2embs[key] for key in project_ids]
    merge_embs = np.vstack((embedd_tarmos, np.vstack((ee_outs, embedd_lv))))
    merge_proj = project_data(merge_embs)
    delf_len = len(embedd_lv) #.shape[0]
    main_len = len(ee_outs)#.shape[0]
    tarmo_len = embedd_tarmos.shape[0]
    tarmo_list = project_data(embedd_tarmos)
    #colors = ["orange"] * tarmo_list.shape[0] + ["red"] * main_len + ['blue'] * delf_len
    plt.xlabel('dimension 1')  
    plt.ylabel('dimension 2')  
    plt.title("Latvian document space - k @ " + str(parameter)) 
    #sizes = [150] * (main_len + tarmo_list.shape[0]) + [5] * delf_len
    #plt.legend(loc=0, prop={'tarmo' : 'orange', 'elaine' : 'orange', 'lv_articles' : 'blue'}) 
    plt.scatter(merge_proj[:tarmo_len, 0], merge_proj[:tarmo_len, 1], s = 8, color = "black", marker = "x", zorder=3, label = "GS query documents")#colors)
    plt.scatter(merge_proj[tarmo_len:(main_len + tarmo_len) , 0], merge_proj[tarmo_len:(main_len + tarmo_len), 1], s = 3, color = "#bdbdbd", zorder=2, label = "interesting seed documents")
    plt.scatter(merge_proj[(main_len + tarmo_len):, 0], merge_proj[(main_len + tarmo_len):, 1], s = 1, color = "#636363", zorder=1, label = "other latvian documents")
    plt.legend(loc='best', fontsize=8)
    plt.savefig("outs_"+str(parameter)+".pdf")
project2(embedd_tarmos, delfi_embed, ids2embs, parameter = 5)
project2(embedd_tarmos, delfi_embed, ids2embs, parameter = 10)
project2(embedd_tarmos, delfi_embed, ids2embs, parameter = 20)
project2(embedd_tarmos, delfi_embed, ids2embs, parameter = 50)
project2(embedd_tarmos, delfi_embed, ids2embs, parameter = 100)
estonian2latvian = pickle.load(open("../pickles/estonian_to_latvian_sbert_rankings.pkl","rb"))
from scipy import spatial
def find_closest_k(tarmos,  delfi_embed, ids2embs, parameter = 5):
    project_ids = set([x for delf in delfi_to_ee for x in delfi_to_ee[delf][:parameter]])
    ee_outs = [ids2embs[key] for key in project_ids]
    tree = spatial.KDTree(ee_outs)
    for tarmo in tarmos:
        #for elaine in ee_outs:
        print(tree.query(tarmo))<jupyter_output><empty_output><jupyter_text>find_closest_k(embedd_tarmos, delfi_embed, ids2embs, parameter = 5)<jupyter_code>find_closest_k(embedd_tarmos, delfi_embed, ids2embs, parameter = 10)
find_closest_k(embedd_tarmos, delfi_embed, ids2embs, parameter = 50)
find_closest_k(embedd_tarmos, delfi_embed, ids2embs, parameter = 100)
from scipy import spatial
def find_closest_k_blue(tarmos, parameter = 5):
    tree = spatial.KDTree(embedd_lv)
    for tarmo in tarmos:
        print(tree.query(tarmo))   
        
        
find_closest_k_blue(embedd_tarmos, 5)
find_closest_k(embedd_tarmos, delfi_embed, ids2embs, parameter = 5)
from scipy import spatial
def find_closest_k_blue(tarmos, parameter = 5):
    tree = spatial.KDTree(embedd_lv)
    for tarmo in tarmos:
        #for elaine in ee_outs:
        print(tree.query(tarmo))           
from scipy import spatial
def find_closest_k_n(tarmos,  delfi_embed, ids2embs, parameter = 5):
    project_ids = set([x for delf in delfi_to_ee for x in delfi_to_ee[delf][:parameter]])
    ee_outs = [ids2embs[key] for key in project_ids]
    len_red_space = len(ee_outs)
    len_blue_space = len(embedd_lv)
    merged = np.vstack((ee_outs, embedd_lv))
    tree = spatial.KDTree(merged)
    k_list = []
    for k in range(3,100):
        ratios = []
        red_hits = 0
        blue_hits = 0
        for tarmo in tarmos:
            #for elaine in ee_outs:
            dist, ind = tree.query(tarmo,k)
            for i in ind:
                if i <= len_red_space:
                    red_hits = red_hits + 1
                else:
                    blue_hits = blue_hits + 1
            blaz_ratio = red_hits / (red_hits+blue_hits)
            ratios.append(blaz_ratio)
        avg_ratio_at_k = sum(ratios)/len(ratios)
        k_list.append(avg_ratio_at_k)
    plt.title('SNIR scores (k=100 and varying size of neighbourhood m)')
    plt.xlabel('m-retrieved neighbours')  
    plt.ylabel('SNIR') 
    plt.plot(list(range(3,100)), k_list)
        
find_closest_k_n(embedd_tarmos, delfi_embed, ids2embs, parameter = 5)
find_closest_k_n(embedd_tarmos, delfi_embed, ids2embs, parameter = 10)
find_closest_k_n(embedd_tarmos, delfi_embed, ids2embs, parameter = 20)
find_closest_k_n(embedd_tarmos, delfi_embed, ids2embs, parameter = 50)
find_closest_k_n(embedd_tarmos, delfi_embed, ids2embs, parameter = 100)
import datetime
date_last_7_f = '2019-11-30T07:00:00.000Z'
date_last_7 = datetime.datetime.strptime(date_last_7_f, '%Y-%m-%dT%H:%M:%S.%fZ')
json_files = ["../articles/lv_2019.json"]
parse_lasts = []
titles_lasts = []
ids_lasts = []
for jf in json_files:
    print("JSON file:", jf)
    articles = json.load(open(jf, 'r', encoding='utf-8'))
    print("Articles:", len(articles))
    for art in articles:
        timez = datetime.datetime.strptime(art["publishDate"], '%Y-%m-%dT%H:%M:%S.%fZ')
        if timez >= date_last_7 and art["channelLanguage"] == 'nat':   
            texts = art['title'].lower() + ' ' + art['bodyText'].lower()
            parse_lasts.append(texts)
            titles_lasts.append(art['title'].lower())
            ids_lasts.append(art['id'])
embedd_last_7 = embedd_texts(parse_lasts)
filez = json.load(open("../articles/21_pairs/articles_21.json", encoding='utf-8'))
tarmos21 = []
for file in filez:
    tarmos21.append(file['lv']['title'] + " "+ file['lv']['body'])
    #tmp_dat = file['']
    #tarmos21.append(file)
    
embedd_tarmos = embedd_texts(tarmos21)

from scipy import spatial
import random
def find_closest_k_n_2_cols(tarmos,  delfi_embed, ids2embs, parameter = 5):
    project_ids = set([x for delf in delfi_to_ee for x in delfi_to_ee[delf][:parameter]])
    ee_outs = [ids2embs[key] for key in project_ids]
    len_red_space = len(ee_outs)
    len_blue_space = len(embedd_lv)
    merged = np.vstack((ee_outs, embedd_lv))
    tree = spatial.KDTree(merged)
    random_pivots = random.choices(embedd_lv, k = 21)
    plt.title('SNIR scores (k=100 and varying size of neighbourhood m)')
    plt.xlabel('m-retrieved neighbours')  
    plt.ylabel('SNIR') 

    for name, pivot in [("GS query documents",tarmos), ("random documents", random_pivots)]:
        k_list = []
        for k in range(3,100):
            ratios = []
            red_hits = 0
            blue_hits = 0
            for tarmo in pivot:
                #for elaine in ee_outs:
                dist, ind = tree.query(tarmo,k)
                for i in ind:
                    if i <= len_red_space:
                        red_hits = red_hits + 1
                    else:
                        blue_hits = blue_hits + 1
                blaz_ratio = red_hits / (red_hits+blue_hits)
                ratios.append(blaz_ratio)
            avg_ratio_at_k = sum(ratios)/len(ratios)
            k_list.append(avg_ratio_at_k)
        plt.plot(list(range(3,100)), k_list, label = name)
    plt.legend(loc='best')
    plt.savefig('qr_eval.pdf')
find_closest_k_n_2_cols(embedd_tarmos, delfi_embed, ids2embs, parameter = 100)
from scipy import spatial
def find_closest_fixed(tarmos,  delfi_embed, ids2embs, titles, parameter = 5, k = 20):
    project_ids = set([x for delf in delfi_to_ee for x in delfi_to_ee[delf][:parameter]])
    ee_outs = [ids2embs[key] for key in project_ids]
    len_red_space = len(ee_outs)
    len_blue_space = len(embedd_lv) 
    merged = np.vstack((ee_outs, embedd_lv))
    tree = spatial.KDTree(merged)
    k_list = []
    ratios = []
    red_hits = 0
    blue_hits = 0
    for title, text, ids, tarmo in zip(titles,parse_lasts,ids_lasts, tarmos):
        #for elaine in ee_outs:
        dist, ind = tree.query(tarmo,k)
        for i in ind:
            if i <= len_red_space:
                red_hits = red_hits + 1
            else:
                blue_hits = blue_hits + 1
        blaz_ratio = red_hits / (red_hits+blue_hits)
        
        ratios.append((title, text, ids, blaz_ratio))
    return ratios
out_df = find_closest_fixed(embedd_last_7, delfi_embed, ids2embs, titles_lasts, parameter = 20, k = 100)
import pandas as pd
x = pd.DataFrame(out_df)
x["id"] = x[2]
x["text"] = x[1]
x["score"] = x[3]
del x[0]
del x[1]
del x[2]
del x[3]
x = x.sort_values(by=['score'], ascending=False)
x["score"].describe()
x<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13322368421052633, 'fraction_numerical': 0.020429362880886426, 'mean_word_length': 3.6064593301435406, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '156669', 'license': 'MIT', 'n_tokens_mistral': 4640, 'n_tokens_neox': 4430, 'n_words': 856, 'path': 'scripts/sBERT+umap-Copy1.ipynb', 'repo_name': 'bkolosk1/Interesting-cross-border-news-discovery', 'size': '11552'}