<jupyter_start><jupyter_text>Preprocessing and Feature Creation In this notebook we import the data, preprocess the data and create features for supervised and unsupervised cross-lingual-information retrieval models.  I. Import Data In this section we import the English and German europarl datasets and combine them into a parallel sentence translation dataframe.<jupyter_code>import os
import sys
sys.path.append(os.path.dirname((os.path.abspath(''))))

from src.data import create_data_subset
create_data_subset(sentence_data_source_path='../data/external/europarl-v7.de-en.en',
                   sentence_data_target_path='../data/external/europarl-v7.de-en.de',
                   sample_size=35000,
                   sentence_data_sampled_path="../data/interim/europarl_en_de_test.pkl",)<jupyter_output>Finished function: 'load_doc' in 1.56 seconds.
Finished function: 'to_sentences' in 1.05 seconds.
Finished function: 'load_doc' in 2.01 seconds.
Finished function: 'to_sentences' in 1.24 seconds.
Sampled dataframe saved in: ../data/interim/europarl_en_de_test.pkl
Finished function: 'create_data_subset' in 7.85 seconds.<jupyter_text>II. Preprocess data In this section we preprocess the parallel sentence data for the feature generation<jupyter_code>import spacy
from nltk.corpus import stopwords
from textblob import TextBlob as textblob_source
from textblob_de import TextBlobDE as textblob_target
import en_core_web_sm
import de_core_news_sm
# import it_core_news_sm
# import pl_core_news_sm
import time
from src.data import PreprocessingEuroParl
stopwords_source = stopwords.words('english')
stopwords_target = stopwords.words('german') # German stopwords
# stopwords_target = stopwords.words('italian') # Italian stopwords
# stopwords_target = stopwords.words('polish') # Polish stopwords
nlp_source = en_core_web_sm.load()
nlp_target = de_core_news_sm.load() # German pipeline
# nlp_target = it_core_news_sm.load() # Italian pipeline
# nlp_target = pl_core_news_sm.load() # Polish pipeline
parallel_sentences = PreprocessingEuroParl(df_sampled_path="../data/interim/europarl_en_de_test.pkl") # German
# parallel_sentences = PreprocessingEuroParl(df_sampled_path="../data/interim/europarl_en_it.pkl") # Italien
# parallel_sentences = PreprocessingEuroParl(df_sampled_path="../data/interim/europarl_en_pol.pkl") # Polnisch
import numpy as np
parallel_sentences.dataframe = parallel_sentences.dataframe.iloc[30000:, :]
parallel_sentences.dataframe = parallel_sentences.dataframe.reset_index(drop=True)
parallel_sentences.dataframe["id_source"] = np.arange(len(parallel_sentences.dataframe))
parallel_sentences.dataframe["id_target"] = np.arange(len(parallel_sentences.dataframe))
parallel_sentences.preprocess_sentences(nlp_source, nlp_target, stopwords_source, stopwords_target)


parallel_sentences.extract_sentence_information(nlp_source, nlp_target)
parallel_sentences.create_embedding_information("proc_5k")
parallel_sentences.create_embedding_information("proc_b_1k")
parallel_sentences.create_embedding_information("vecmap")
parallel_sentences.preprocessed.to_json("../data/interim/preprocessed_data_en_de_testset.json")
parallel_sentences.dataframe
parallel_sentences.preprocessed
import pandas as pd
preprocessed_data = pd.read_json("../data/interim/preprocessed_data_en_de_testset.json")
parallel_sentences = PreprocessingEuroParl(df_sampled_path="../data/interim/europarl_en_de_test.pkl")
parallel_sentences.preprocessed = preprocessed_data<jupyter_output><empty_output><jupyter_text>III. Create data set In this section we create the datasets for the training of the supervised model and the data for the supervised and unsupervised retrieval.<jupyter_code>from src.data import DataSet
n_model = 0
n_queries = 100
n_retrieval = 5000
k = 10
sample_size_k = 100

dataset = DataSet(parallel_sentences.preprocessed)
#dataset = DataSet(preprocessed_data)
dataset.split_model_retrieval(n_model, n_retrieval)
import pandas as pd
#dataset.create_retrieval_index(n_queries)

# If your pandas version is old, use this instead
query = pd.DataFrame({"id_source": dataset.retrieval_subset.iloc[:n_queries]["id_source"]})
documents = pd.DataFrame({"id_target": dataset.retrieval_subset["id_target"]})
index = pd.MultiIndex.from_product([dataset.retrieval_subset.iloc[:n_queries]["id_source"], dataset.retrieval_subset["id_target"]], names = ["id_source", "id_target"])
dataset.retrieval_dataset_index = pd.DataFrame(index = index).reset_index()
dataset.retrieval_dataset_index.reset_index(drop=True).to_feather("../data/processed/dataset_retrieval_index_en_de_testset.feather")
# import pandas as pd
# pd.read_feather("../data/processed/dataset_retrieval_index.feather")<jupyter_output><empty_output><jupyter_text>IV. Create features In this section we create features for our model, that are sentence based and should be created before the text is preprocessed.<jupyter_code>#%autoreload 2
from src.features import feature_generation_class<jupyter_output><empty_output><jupyter_text>Generation of the data for the crosslingual information retrieval task.<jupyter_code>features_retrieval = feature_generation_class.FeatureGeneration(dataset.retrieval_dataset_index, 
                                                            parallel_sentences.preprocessed)
features_retrieval.create_feature_dataframe()
features_retrieval.create_sentence_features()
features_retrieval.create_embedding_features("proc_5k")
features_retrieval.create_embedding_features("proc_b_1k")
features_retrieval.create_embedding_features("vecmap")
features_retrieval.feature_dataframe.reset_index(drop=True).to_feather("../data/processed/feature_retrieval_en_de_testset.feather")
# import pandas as pd
# pd.read_feather("../data/processed/feature_retrieval.feather")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0911449325492909, 'fraction_numerical': 0.007782774126599792, 'mean_word_length': 7.202836879432624, 'pattern_counts': {'":': 2, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '524211', 'license': 'MIT', 'n_tokens_mistral': 1893, 'n_tokens_neox': 1816, 'n_words': 341, 'path': 'notebooks/Preprocessing_Feature_Generation_en_de_testset.ipynb', 'repo_name': 'JakobLangenbahn/crosslingual-information-retrieval', 'size': '5782'}