<jupyter_start><jupyter_text>All imports necessary<jupyter_code>import os
import sys
import lightgbm as lgb
import xgboost as xgb
import warnings
warnings.filterwarnings("ignore")
import numpy as np
import pandas as pd
import multiprocessing
sys.path.append('..')
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import auc
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import KBinsDiscretizer
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.compose import ColumnTransformer
from sklearn.metrics import roc_curve
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from source.code.transformers.metafeaturesextractor import MetaFeaturesExtractor
from source.code.models.svdbasedrecommender import SVDBasedRecommender
from source.code.models.songfrequencybasedrecommender import SongFrequencyBasedRecommender
from source.code.models.factorizationmachinebasedrecommender import FactorizationMachineBasedRecommender
from source.code.models.lightgbmbasedrecommender import LightGBMBasedRecommender
from source.code.models.xgboostbasedrecommender import XGBoostBasedRecommender
from source.code.models.ensemblebasedrecommender import EnsembleBasedRecommender
from tqdm.autonotebook import tqdm<jupyter_output><empty_output><jupyter_text>Auxiliary methods<jupyter_code>def plot_roc_auc(fpr, tpr, classifier):
    plt.figure(figsize=(10, 10))
    plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.10f)' % auc(fpr, tpr))
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.01])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic example ({})'.format(classifier))
    plt.legend(loc="lower right")
    plt.show()<jupyter_output><empty_output><jupyter_text>Read the data<jupyter_code>data_directory = '../data/datasets/'
unknown_category_label = 'unknown'
unknown_language_label = -1<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>train = pd.read_csv(os.path.join(data_directory, 'train.csv'), engine='python')
train.head(4).T
train.info(verbose=True, null_counts=True)<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 7377418 entries, 0 to 7377417
Data columns (total 6 columns):
msno                  7377418 non-null object
song_id               7377418 non-null object
source_system_tab     7352569 non-null object
source_screen_name    6962614 non-null object
source_type           7355879 non-null object
target                7377418 non-null int64
dtypes: int64(1), object(5)
memory usage: 337.7+ MB<jupyter_text>Test<jupyter_code>test = pd.read_csv(os.path.join(data_directory, 'test.csv'), engine='python')
test.head(4).T
test.info(verbose=True, null_counts=True)<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 2556790 entries, 0 to 2556789
Data columns (total 6 columns):
id                    2556790 non-null int64
msno                  2556790 non-null object
song_id               2556790 non-null object
source_system_tab     2548348 non-null object
source_screen_name    2393907 non-null object
source_type           2549493 non-null object
dtypes: int64(1), object(5)
memory usage: 117.0+ MB<jupyter_text>Songs<jupyter_code>songs = pd.read_csv(os.path.join(data_directory, 'songs.csv'))
songs.head(4).T
songs.info(verbose=True, null_counts=True)<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 2296320 entries, 0 to 2296319
Data columns (total 7 columns):
song_id        2296320 non-null object
song_length    2296320 non-null int64
genre_ids      2202204 non-null object
artist_name    2296320 non-null object
composer       1224966 non-null object
lyricist       351052 non-null object
language       2296319 non-null float64
dtypes: float64(1), int64(1), object(5)
memory usage: 122.6+ MB<jupyter_text>Members<jupyter_code>members = pd.read_csv(os.path.join(data_directory, 'members.csv'))
members.head(4).T
members.info(verbose=True, null_counts=True)<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 34403 entries, 0 to 34402
Data columns (total 7 columns):
msno                      34403 non-null object
city                      34403 non-null int64
bd                        34403 non-null int64
gender                    14501 non-null object
registered_via            34403 non-null int64
registration_init_time    34403 non-null int64
expiration_date           34403 non-null int64
dtypes: int64(5), object(2)
memory usage: 1.8+ MB<jupyter_text>Sample submission<jupyter_code>sample_submission = pd.read_csv(os.path.join(data_directory, 'sample_submission.csv'))
sample_submission.head(4).T
sample_submission.info(verbose=True, null_counts=True)<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 2556790 entries, 0 to 2556789
Data columns (total 2 columns):
id        2556790 non-null int64
target    2556790 non-null float64
dtypes: float64(1), int64(1)
memory usage: 39.0 MB<jupyter_text>Solutions  Train / Test split preparation<jupyter_code>X, y = train[train.columns[:-1]], train[train.columns[-1]]
X_for_submission = test[test.columns]
X.head(4).T
y.head(4).T
X_for_submission.head(4).T
X[['msno', 'song_id']].values.shape
X_train, X_test, y_train, y_test = train_test_split(
    X,
    y,
    test_size=0.3,
    random_state=42,
    stratify=y
)
print(X_train.shape, y_train.shape)
print(type(X_train), type(y_train))
print(X_test.shape, y_test.shape)
print(type(X_test), type(y_test))<jupyter_output><class 'pandas.core.frame.DataFrame'> <class 'pandas.core.series.Series'><jupyter_text>Preprocessing steps description<jupyter_code>categorical_features = [
    'source_system_tab',
    'source_screen_name',
    'city',
    'gender'
]
categorical_features_lang = [
    'language'
]
numerical_features = [
    'bd',
    'song_length'
]
numerical_features_days_reg = [
    'days_registered'
]
num_features_pipeline = Pipeline([
    ('impute', SimpleImputer(missing_values=np.nan, strategy='mean')),
    ('discretize', KBinsDiscretizer(n_bins=4, encode='onehot-dense'))
])
num_features_pipeline_days_reg = Pipeline([
    ('impute', SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=0)),
    ('discretize', KBinsDiscretizer(n_bins=4, encode='onehot-dense'))
])
cat_features_pipeline = Pipeline([
    ('impute', SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=unknown_category_label)),
    ('onehot', OneHotEncoder(handle_unknown='ignore', sparse=False))
])
cat_features_pipeline_lang = Pipeline([
    ('impute', SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=unknown_language_label)),
    ('onehot', OneHotEncoder(handle_unknown='ignore', sparse=False))
])
preprocessor = ColumnTransformer(
    transformers=[
        ('num', num_features_pipeline, numerical_features),
        ('num_days_reg', num_features_pipeline_days_reg, numerical_features_days_reg),
        ('cat', cat_features_pipeline, categorical_features),
        ('cat_lang', cat_features_pipeline_lang, categorical_features_lang)
    ]
)
unified_pipeline = Pipeline(
    steps=[
        ('add_meta_info', MetaFeaturesExtractor(user_meta=members, item_meta=songs)),
        ('preprocessing', preprocessor)
    ]
)<jupyter_output><empty_output><jupyter_text>Naive frequency approach<jupyter_code>naive_frequency_recommender = SongFrequencyBasedRecommender()
naive_frequency_recommender.fit(X_train, y_train)
y_pred = naive_frequency_recommender.predict_proba(X_test)
fpr, tpr, _ = roc_curve(y_test, y_pred)
plot_roc_auc(fpr, tpr, 'Naive frequency approach')<jupyter_output><empty_output><jupyter_text>SVD<jupyter_code>svd_recommender = SVDBasedRecommender()
svd_recommender.fit(X_train, y_train)
y_pred = svd_recommender.predict_proba(X_test)
fpr, tpr, _ = roc_curve(y_test, y_pred)
plot_roc_auc(fpr, tpr, 'SVD approach')<jupyter_output><empty_output><jupyter_text>FM<jupyter_code>fm_classifier_pipeline = Pipeline(
    steps=[
        ('data_preparation', unified_pipeline),
        ('classification', FactorizationMachineBasedRecommender(show_progress=True))
    ]
)
fm_classifier_pipeline.fit(X_train, y_train.values)
y_pred = fm_classifier_pipeline.predict_proba(X_test)
y_pred.shape
fpr, tpr, _ = roc_curve(y_test, y_pred)
plot_roc_auc(fpr, tpr, 'Factorization machine approach')<jupyter_output><empty_output><jupyter_text>LightGBM<jupyter_code>lgb_classifier_pipeline = Pipeline(
    steps=[
        ('data_preparation', unified_pipeline),
        ('classification', LightGBMBasedRecommender())
    ]
)
lgb_classifier_pipeline.fit(X_train, y_train.values)
y_pred = lgb_classifier_pipeline.predict_proba(X_test)
y_pred.shape
fpr, tpr, _ = roc_curve(y_test, y_pred)
plot_roc_auc(fpr, tpr, 'LightGBM classifier approach')<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>xgb_classifier_pipeline = Pipeline(
    steps=[
        ('data_preparation', unified_pipeline),
        ('classification', XGBoostBasedRecommender())
    ]
)
xgb_classifier_pipeline.fit(X_train, y_train.values)
y_pred = xgb_classifier_pipeline.predict_proba(X_test)
y_pred.shape
fpr, tpr, _ = roc_curve(y_test, y_pred)
plot_roc_auc(fpr, tpr, 'XGBoost classifier approach')<jupyter_output><empty_output><jupyter_text>StarSpace<jupyter_code>star_space_num_features_pipeline = Pipeline([
    ('impute', SimpleImputer(missing_values=np.nan, strategy='mean')),
    ('discretize', KBinsDiscretizer(n_bins=4, encode='ordinal'))
])
star_space_num_features_pipeline_days_reg = Pipeline([
    ('impute', SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=0)),
    ('discretize', KBinsDiscretizer(n_bins=4, encode='ordinal'))
])
star_space_cat_features_pipeline = Pipeline([
    ('impute', SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=unknown_category_label)),
    ('encode', OrdinalEncoder())
])
star_space_cat_features_pipeline_lang = Pipeline([
    ('impute', SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=unknown_language_label)),
    ('encode', OrdinalEncoder())
])
star_space_preprocessor = ColumnTransformer(
    transformers=[
        ('num', star_space_num_features_pipeline, numerical_features),
        ('num_days_reg', star_space_num_features_pipeline_days_reg, numerical_features_days_reg),
        ('cat', star_space_cat_features_pipeline, categorical_features),
        ('cat_lang', star_space_cat_features_pipeline_lang, categorical_features_lang)
    ]
)
star_space_unified_pipeline = Pipeline(
    steps=[
        ('add_meta_info', MetaFeaturesExtractor(user_meta=members, item_meta=songs)),
        ('preprocessing', star_space_preprocessor)
    ]
)
X_star_space = star_space_unified_pipeline.fit_transform(X, y)
X.shape
numerical_features + numerical_features_days_reg + categorical_features + categorical_features_lang
X_star_space.shape
X_star_space = pd.DataFrame(X_star_space, columns=numerical_features + numerical_features_days_reg + categorical_features + categorical_features_lang)
X_star_space.head()
X_star_space.info(verbose=True, null_counts=True)
X_star_space.bd = X_star_space.bd.astype(np.int32)
X_star_space.song_length = X_star_space.song_length.astype(np.int32)
X_star_space.days_registered = X_star_space.days_registered.astype(np.int32)
X_star_space.source_system_tab = X_star_space.source_system_tab.astype(np.int32)
X_star_space.source_screen_name = X_star_space.source_screen_name.astype(np.int32)
X_star_space.city = X_star_space.city.astype(np.int32)
X_star_space.gender = X_star_space.gender.astype(np.int32)
X_star_space.language = X_star_space.language.astype(np.int32)
X_star_space.info(verbose=True, null_counts=True)
curr = 0
bd_map = dict(zip(X_star_space.bd.unique(), range(curr, curr + len(X_star_space.bd.unique()))))
curr += len(X_star_space.bd.unique())
song_length_map = dict(zip(X_star_space.song_length.unique(), range(curr, curr + len(X_star_space.song_length.unique()))))
curr += len(X_star_space.song_length.unique())
days_registered_map = dict(zip(X_star_space.days_registered.unique(), range(curr, curr + len(X_star_space.days_registered.unique()))))
curr += len(X_star_space.days_registered.unique())
source_system_tab_map = dict(zip(X_star_space.source_system_tab.unique(), range(curr, curr + len(X_star_space.source_system_tab.unique()))))
curr += len(X_star_space.source_system_tab.unique())
source_screen_name_map = dict(zip(X_star_space.source_screen_name.unique(), range(curr, curr + len(X_star_space.source_screen_name.unique()))))
curr += len(X_star_space.source_screen_name.unique())
city_map = dict(zip(X_star_space.city.unique(), range(curr, curr + len(X_star_space.city.unique()))))
curr += len(X_star_space.city.unique())
gender_map = dict(zip(X_star_space.gender.unique(), range(curr, curr + len(X_star_space.gender.unique()))))
curr += len(X_star_space.gender.unique())
language_map = dict(zip(X_star_space.language.unique(), range(curr, curr + len(X_star_space.language.unique()))))
bd_map
song_length_map
days_registered_map
source_system_tab_map
source_screen_name_map
city_map
gender_map
language_map
X_star_space.bd = X_star_space.bd.map(bd_map)
X_star_space.song_length = X_star_space.song_length.map(song_length_map)
X_star_space.days_registered = X_star_space.days_registered.map(days_registered_map)
X_star_space.source_system_tab = X_star_space.source_system_tab.map(source_system_tab_map)
X_star_space.source_screen_name = X_star_space.source_screen_name.map(source_screen_name_map)
X_star_space.city = X_star_space.city.map(city_map)
X_star_space.gender = X_star_space.gender.map(gender_map)
X_star_space.language = X_star_space.language.map(language_map)
X_star_space.head()
X_star_space.info(verbose=True, null_counts=True)
X_star_space.to_csv(os.path.join('/mnt/PROMO_INVESTIGATION_DATA/', 'X_star_space.csv'), index=False, header=False, sep=' ')
# Code for StarSpace installation:
# TBD on new machine
! ../../Starspace/starspace
! ../../Starspace/starspace train \
-trainFile /mnt/PROMO_INVESTIGATION_DATA/X_star_space.csv \
-model /mnt/PROMO_INVESTIGATION_DATA/star_space_model \
-trainMode 5
X_star_space_emb = pd.read_csv(os.path.join('/mnt/PROMO_INVESTIGATION_DATA/', 'star_space_model.tsv'), delimiter='\t', header=None)
X_star_space_emb.head()
# SSF stands for StarSpace Feature
X_star_space_emb.columns = ['SSF_{}'.format(i) for i in range(101)]
X_star_space_emb.head()
X_star_space_emb.rename({"SSF_0": "WORD"}, axis=1, inplace=True)
X_star_space_emb.head()
X_star_space_emb.info(verbose=True, null_counts=True)
X_star_space_emb.set_index('WORD', inplace=True)
X_star_space_emb.head()
X_star_space.loc[0]
X_star_space_emb.loc[X_star_space.loc[4]]
def do_something(number):
    return X_star_space_emb.loc[X_star_space.loc[number]].mean()
with multiprocessing.Pool(multiprocessing.cpu_count()) as p:
    X_star_space_emb_extr = pd.concat(p.map(do_something, range(X_star_space.shape[0])), axis=1)
X_star_space_emb_extr = X_star_space_emb_extr.transpose()
X_star_space_emb_extr.head()
X_star_space_emb_extr.info(verbose=True, null_counts=True)
starspace_features = X_star_space_emb_extr.columns
starspace_features = starspace_features.values.tolist()
X_star_space_emb_extr = pd.concat([X[['msno', 'song_id']], X_star_space_emb_extr], axis=1)
X_star_space_emb_extr.to_csv(os.path.join('/mnt/PROMO_INVESTIGATION_DATA/', 'X_star_space_emb_extr.csv'))
X_star_space_emb_extr = pd.read_csv(os.path.join('/mnt/PROMO_INVESTIGATION_DATA/', 'X_star_space_emb_extr.csv'))
from sklearn.base import BaseEstimator, TransformerMixin
starspace_features = ['SSF_{}'.format(i) for i in range(1, 51)]
class StarSpaceFeaturesExtractor(BaseEstimator, TransformerMixin):
    def __init__(self, star_space_meta):
        self.star_space_meta = star_space_meta
        self.X_with_meta = None

    def fit(self, X, y=None, **fit_params):
        return self

    def transform(self, X):
        self.X_with_meta = X.copy()
        self.X_with_meta = pd.merge(self.X_with_meta, self.star_space_meta, on=['msno', 'song_id'], how='left')
        return self.X_with_meta
star_space_preprocessor = ColumnTransformer(
    transformers=[
        ('num', num_features_pipeline, numerical_features + starspace_features),
        ('num_days_reg', num_features_pipeline_days_reg, numerical_features_days_reg),
        ('cat', cat_features_pipeline, categorical_features),
        ('cat_lang', cat_features_pipeline_lang, categorical_features_lang)
    ]
)
star_space_unified_pipeline = Pipeline(
    steps=[
        ('add_meta_info', MetaFeaturesExtractor(user_meta=members, item_meta=songs)),
        ('add_star_space_features', StarSpaceFeaturesExtractor(star_space_meta=X_star_space_emb_extr)),
        ('preprocessing', star_space_preprocessor)
    ]
)
star_space_lgb_classifier_pipeline = Pipeline(
    steps=[
        ('data_preparation', star_space_unified_pipeline),
        ('classification', LightGBMBasedRecommender())
    ]
)
star_space_lgb_classifier_pipeline.fit(X_train, y_train.values)
y_pred = star_space_lgb_classifier_pipeline.predict_proba(X_test)
y_pred.shape
fpr, tpr, _ = roc_curve(y_test, y_pred)
plot_roc_auc(fpr, tpr, 'LightGBM classifier on original features + StarSpace embeddings approach')<jupyter_output><empty_output><jupyter_text>Ensemble<jupyter_code>ens_classifier_pipeline = Pipeline(
    steps=[
        ('data_preparation', unified_pipeline),
        ('classification', EnsembleBasedRecommender(
            base_estimators=[
                FactorizationMachineBasedRecommender(),
                LightGBMBasedRecommender(),
                XGBoostBasedRecommender()
            ]
        ))
    ]
)
ens_classifier_pipeline.fit(X_train, y_train.values)
y_pred = ens_classifier_pipeline.predict_proba(X_test)
y_pred.shape
fpr, tpr, _ = roc_curve(y_test, y_pred)
plot_roc_auc(fpr, tpr, 'Ensemble approach')<jupyter_output><empty_output><jupyter_text>Ensemble + StarSpace features<jupyter_code>star_space_ens_classifier_pipeline = Pipeline(
    steps=[
        ('data_preparation', star_space_unified_pipeline),
        ('classification', EnsembleBasedRecommender(
            base_estimators=[
                FactorizationMachineBasedRecommender(),
                LightGBMBasedRecommender(),
                XGBoostBasedRecommender()
            ]
        ))
    ]
)
star_space_ens_classifier_pipeline.fit(X_train, y_train.values)
y_pred = star_space_ens_classifier_pipeline.predict_proba(X_test)
y_pred.shape
fpr, tpr, _ = roc_curve(y_test, y_pred)
plot_roc_auc(fpr, tpr, 'Ensemble + StarSpace embeddings approach')<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>star_space_ens_classifier_pipeline.fit(X, y.values)
y_pred_subm = ens_classifier_pipeline.predict_proba(X_for_submission)
y_pred_subm.shape
X_for_submission['target'] = y_pred_subm
X_for_submission[['id', 'target']].to_csv(os.path.join(data_directory, 'ensemble_submission_full_train_part.csv'), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11250327653997379, 'fraction_numerical': 0.02070773263433814, 'mean_word_length': 6.283696067201221, 'pattern_counts': {'":': 1, '<': 78, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '484253', 'license': 'Unlicense', 'n_tokens_mistral': 7015, 'n_tokens_neox': 6744, 'n_words': 999, 'path': 'notebooks/SOLUTION.ipynb', 'repo_name': 'IooHooI/RECOMMENDATION_SYSTEMS_2', 'size': '19075'}