<jupyter_start><jupyter_text>Preprocess Data<jupyter_code>with open('/content/drive/MyDrive/crosslingual-information-retrieval/data/interim/europarl_en_de.pkl', "rb") as fh:
    original_sentence_pairs = pickle.load(fh)

feature_retrieval = pd.read_feather("/content/drive/MyDrive/crosslingual-information-retrieval/data/processed/feature_retrieval_en_de.feather")
feature_retrieval = feature_retrieval.rename(columns={"id_source": "source_id", "id_target": "target_id"})
dataset_pairs = get_sentence_pairs(original_sentence_pairs, feature_retrieval)<jupyter_output><empty_output><jupyter_text>Unsupervised: XLM-R First Layer<jupyter_code># Text Encoder First Layer
print("--------------------------------")
print("\nCreate  Text Encoder First Layer model")


xlm_r = TextEncoders("xlm-r")
source_known = {}
source_embedding = []
for sentence in tqdm(dataset_pairs["text_source"].values.tolist()):
    if sentence in source_known.keys():
        source_embedding.append(source_known[sentence])
        continue
    sentence_embedding = xlm_r.create_embedding_for_each_term(sentence, use_layer=1)
    sentence_embedding = np.average(sentence_embedding[0], axis=0)
    source_known[sentence] = sentence_embedding
    source_embedding.append(sentence_embedding)

target_known = {}
target_embedding = []
for sentence in tqdm(dataset_pairs["text_target"].values.tolist()):
    if sentence in target_known.keys():
        target_embedding.append(target_known[sentence])
        continue
    sentence_embedding = xlm_r.create_embedding_for_each_term(sentence, use_layer=1)
    sentence_embedding = np.average(sentence_embedding[0], axis=0)
    target_known[sentence] = sentence_embedding
    target_embedding.append(sentence_embedding)

cosine_similarity_total = []
for i in tqdm(range(len(target_embedding))):
    cos = cosine_similarity(source_embedding[i].reshape(1, -1), target_embedding[i].reshape(1, -1), dense_output=True)[0][0]
    cosine_similarity_total.append(cos)


cosine_similarity_total = [[1-i, i] for i in cosine_similarity_total]
print("The MAP score on test set: {:.4f}".format(MAP_score(feature_retrieval['source_id'],feature_retrieval['Translation'],cosine_similarity_total)))
cosine_similarity_total = []
for i in tqdm(range(len(target_embedding))):
    cos = cosine_similarity(source_embedding[i].reshape(1, -1), target_embedding[i].reshape(1, -1), dense_output=True)[0][0]
    cosine_similarity_total.append(cos)


cosine_similarity_total = [[1-i, i] for i in cosine_similarity_total]
print("The MAP score on test set: {:.4f}".format(MAP_score(feature_retrieval['source_id'],feature_retrieval['Translation'],cosine_similarity_total)))<jupyter_output>The MAP score on test set: 0.0355<jupyter_text>Unsupervised: XLM-R Last Layer<jupyter_code>source_known = {}
source_embedding = []
for sentence in tqdm(dataset_pairs["text_source"].values.tolist()):
    if sentence in source_known.keys():
        source_embedding.append(source_known[sentence])
        continue
    sentence_embedding = xlm_r.create_embedding_for_each_term(sentence, use_layer=1)
    sentence_embedding = np.average(sentence_embedding[0], axis=0)
    source_known[sentence] = sentence_embedding
    source_embedding.append(sentence_embedding)




target_known = {}
target_embedding = []
for sentence in tqdm(dataset_pairs["text_target"].values.tolist()):
    if sentence in target_known.keys():
        target_embedding.append(target_known[sentence])
        continue
    sentence_embedding = xlm_r.create_embedding_for_each_term(sentence, use_layer=12)
    sentence_embedding = np.average(sentence_embedding[0], axis=0)
    target_known[sentence] = sentence_embedding
    target_embedding.append(sentence_embedding)

cosine_similarity_total = []
for i in tqdm(range(len(target_embedding))):
    cos = cosine_similarity(source_embedding[i].reshape(1, -1), target_embedding[i].reshape(1, -1), dense_output=True)[0][0]
    cosine_similarity_total.append(cos)

cosine_similarity_total = [[1-i, i] for i in cosine_similarity_total]
print("The MAP score on test set: {:.4f}".format(MAP_score(feature_retrieval['source_id'],feature_retrieval['Translation'],cosine_similarity_total)))<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`
  This is separate from the ipykernel package so we can avoid doing imports until
{'fraction_non_alphanumeric': 0.12278338945005611, 'fraction_numerical': 0.010774410774410775, 'mean_word_length': 6.51433389544688, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '537086', 'license': 'MIT', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1512, 'n_words': 232, 'path': 'notebooks/Unsupervised_Retrieval_XLMR.ipynb', 'repo_name': 'JakobLangenbahn/crosslingual-information-retrieval', 'size': '4455'}