<jupyter_start><jupyter_text>Supervised Retrieval In this notebook we use the supervised classification model for a supervised crosslingual information retrieval task.<jupyter_code>import sys
import os
sys.path.append(os.path.dirname((os.path.abspath(''))))

import pandas as pd
import numpy as np
import seaborn as sn
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, log_loss
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.neural_network import MLPClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.feature_selection import RFECV
from src.models.predict_model import MAP_score, threshold_counts, feature_selection, pipeline_model_optimization<jupyter_output><empty_output><jupyter_text>I. Import Data In this section we import the feature dataframe for the retrieval task.<jupyter_code>feature_dataframe=pd.read_feather("../data/processed/feature_model_en_de.feather")
feature_retrieval=pd.read_feather("../data/processed/feature_retrieval_en_de.feather")
feature_dataframe = feature_dataframe.rename(columns={"id_source": "source_id", "id_target": "target_id"})
feature_retrieval = feature_retrieval.rename(columns={"id_source": "source_id", "id_target": "target_id"})


# Load Test Data
feature_retrieval_de = pd.read_feather("../data/processed/feature_retrieval_en_de_testset.feather")
feature_retrieval_de = feature_retrieval_de.rename(columns={"id_source": "source_id", "id_target": "target_id"})

feature_retrieval_pl = pd.read_feather("../data/processed/feature_retrieval_en_pl.feather")
feature_retrieval_pl = feature_retrieval_pl.rename(columns={"id_source": "source_id", "id_target": "target_id"})

feature_retrieval_it = pd.read_feather("../data/processed/feature_retrieval_en_it.feather")
feature_retrieval_it = feature_retrieval_it.rename(columns={"id_source": "source_id", "id_target": "target_id"})

feature_retrieval_doc = pd.read_feather("../data/processed/feature_retrieval_doc.feather")
feature_retrieval_doc = feature_retrieval_doc.rename(columns={"id_source": "source_id", "id_target": "target_id"})<jupyter_output><empty_output><jupyter_text>Delete all columns with only one value<jupyter_code>column_mask = feature_dataframe.apply(threshold_counts, threshold=1)
feature_dataframe = feature_dataframe.loc[:, column_mask]
feature_retrieval = feature_retrieval.loc[:, column_mask]<jupyter_output><empty_output><jupyter_text>II. Supervised Retrieval  Start with one feature<jupyter_code>start_features = ["jaccard_translation_proc_5k"]
not_add = ["Translation", "source_id", "target_id"]
added_features = feature_dataframe.columns[~feature_dataframe.columns.isin(start_features+not_add)]<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>from xgboost import XGBClassifier

start_features = ["jaccard_translation_proc_5k"]
not_add = ["Translation", "source_id", "target_id"]
added_features = feature_dataframe.columns[~feature_dataframe.columns.isin(start_features+not_add)]

xgb = XGBClassifier(verbosity = 0, use_label_encoder=False, random_state=42)
scaler = preprocessing.StandardScaler()

xgb_parameter_grid = {"verbosity": [0],
                      "use_label_encoder": [False],
                      "random_state": [42],
                      'min_child_weight': [1, 5],
                      'gamma': [0.5, 1,  5],
                      'subsample': [0.6, 0.8],
                      'colsample_bytree': [0.6, 0.8, 1.0],
                      'max_depth': [3, 4, 5]}

xgb_best_features, xgb_best_parameter_combination, xgb_best_map_score, xgb_all_parameter_combination = \
pipeline_model_optimization(xgb, xgb_parameter_grid, scaler, feature_dataframe, 
                            feature_retrieval, start_features, 
                            added_features, 
                            threshold_map_feature_selection=0.001)
xgb_best_features
xgb_best_features_ = ['jaccard_translation_proc_5k',
 'cosine_similarity_tf_idf_vecmap',
 'cosine_similarity_average_vecmap',
 'number_]_difference_normalized',
 'number_?_difference_normalized',
 'number_-_difference_relative',
 'number_[_difference_normalized',
 'jaccard_numbers_source']

xgb_best_hyperparameters = {'verbosity': 0, 'use_label_encoder': False, 'random_state': 42, 'min_child_weight': 1, 'gamma': 1, 'subsample': 0.6, 'colsample_bytree': 0.6, 'max_depth': 3}<jupyter_output><empty_output><jupyter_text>Evaluate on Test Set<jupyter_code>from xgboost import XGBClassifier

target_train=feature_dataframe['Translation'].astype(float)
data_train=feature_dataframe.drop(columns=['Translation','source_id','target_id'])
data_train = data_train.loc[:, xgb_best_features_]
scaler = preprocessing.StandardScaler()
data_train.loc[:, data_train.columns] = scaler.fit_transform(data_train.loc[:, data_train.columns])

print("Model was trained on EN-DE Parallel Sentences.\n")
xgb_classifier = XGBClassifier(**xgb_best_hyperparameters).fit(data_train.to_numpy(), target_train.to_numpy())

# EN-DE
target_test = feature_retrieval_de['Translation'].astype(float)
data_test = feature_retrieval_de.drop(columns=['Translation','source_id','target_id'])
data_test = data_test.loc[:, xgb_best_features_]
data_test.loc[:, data_test.columns] = scaler.transform(data_test.loc[:, data_test.columns])
prediction = xgb_classifier.predict_proba(data_test).tolist()
print("EN-DE Map Score: {}".format(MAP_score(feature_retrieval_de['source_id'],target_test,prediction)))

# EN-IT
target_test = feature_retrieval_it['Translation'].astype(float)
data_test = feature_retrieval_it.drop(columns=['Translation','source_id','target_id'])
data_test = data_test.loc[:, xgb_best_features_]
data_test.loc[:, data_test.columns] = scaler.transform(data_test.loc[:, data_test.columns])
prediction = xgb_classifier.predict_proba(data_test).tolist()
print("EN-IT Map Score: {}".format(MAP_score(feature_retrieval_it['source_id'],target_test,prediction)))

# EN-PL
target_test = feature_retrieval_pl['Translation'].astype(float)
data_test = feature_retrieval_pl.drop(columns=['Translation','source_id','target_id'])
data_test = data_test.loc[:, xgb_best_features_]
data_test.loc[:, data_test.columns] = scaler.transform(data_test.loc[:, data_test.columns])
prediction = xgb_classifier.predict_proba(data_test).tolist()
print("EN-PL Map Score: {}".format(MAP_score(feature_retrieval_pl['source_id'],target_test,prediction)))

# Document Corpus
target_test = feature_retrieval_doc['Translation'].astype(float)
data_test = feature_retrieval_doc.drop(columns=['Translation','source_id','target_id'])
data_test = data_test.loc[:, xgb_best_features_]
data_test.loc[:, data_test.columns] = scaler.transform(data_test.loc[:, data_test.columns])
prediction = xgb_classifier.predict_proba(data_test).tolist()
print("EN-PL Map Score: {}".format(MAP_score(feature_retrieval_doc['source_id'],target_test,prediction)))<jupyter_output>Model was trained on EN-DE Parallel Sentences.

EN-DE Map Score: 0.8431334195285983
EN-IT Map Score: 0.8686442751667981
EN-PL Map Score: 0.8664685384247898
EN-PL Map Score: 0.0021624042275190146
{'fraction_non_alphanumeric': 0.12123325892857142, 'fraction_numerical': 0.016043526785714284, 'mean_word_length': 7.414319248826291, 'pattern_counts': {'":': 15, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '59573', 'license': 'MIT', 'n_tokens_mistral': 2498, 'n_tokens_neox': 2415, 'n_words': 364, 'path': 'notebooks/Supervised_Retrieval_XGBoost.ipynb', 'repo_name': 'JakobLangenbahn/crosslingual-information-retrieval', 'size': '7168'}