<jupyter_start><jupyter_text>Download libraries and data<jupyter_code>import pandas as pd
import numpy as np
import copy
from itertools import combinations

from lightgbm import LGBMClassifier
from xgboost import XGBClassifier

from sklearn.metrics import log_loss
from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold
from sklearn.preprocessing import LabelEncoder

from tqdm import tqdm, tqdm_notebook

pd.set_option('display.max_columns', 100)

import warnings
warnings.filterwarnings("ignore")
import time
train = pd.read_csv('Train.csv')
test = pd.read_csv('Test.csv')
sub = pd.read_csv('SampleSubmission.csv')<jupyter_output><empty_output><jupyter_text>Checking the data<jupyter_code>train.head(2)
test.head(2)
sub.head(2)<jupyter_output><empty_output><jupyter_text>Replacing non common occupation codes to occupation category code<jupyter_code>replace_train = list(
    set(train['occupation_code'].unique().tolist()) -
    set(test['occupation_code']))
replace_test = list(
    set(test['occupation_code'].unique().tolist()) -
    set(train['occupation_code']))

train['occupation_code'] = train['occupation_code'].replace(
    replace_train, np.nan)
test['occupation_code'] = test['occupation_code'].replace(replace_test, np.nan)
train['occupation_code'].fillna(train['occupation_category_code'],
                                inplace=True)
test['occupation_code'].fillna(test['occupation_category_code'], inplace=True)<jupyter_output><empty_output><jupyter_text>Getting right format<jupyter_code>#Make spliting train clients info. Trying to reproduce the situation with test
#
X_train = []
X_train_columns = train.columns[:-1]
client_index = 0

for line in tqdm_notebook(train.values):

    info = line[:8]
    info_products = line[8:-1]
    indexes = [k for k, i in enumerate(info_products) if i == 1]

    for i in indexes:

        client_index += 1

        for k in range(len(info_products)):

            if k == i:

                info_products_transformed = list(copy.copy(info_products))
                info_products_transformed[i] = 0

                X_train.append(
                    list(info) + info_products_transformed +
                    [X_train_columns[8 + k]] + [client_index])

X_train = pd.DataFrame(X_train)
X_train.columns = [
    'ID', 'join_date', 'sex', 'marital_status', 'birth_year', 'branch_code',
    'occupation_code', 'occupation_category_code', 'P5DA', 'RIBP', '8NN1',
    '7POT', '66FJ', 'GYSR', 'SOP4', 'RVSZ', 'PYUQ', 'LJR9', 'N2MW', 'AHXO',
    'BSTQ', 'FM3X', 'K6QO', 'QBOL', 'JWFN', 'JZ9D', 'J9JW', 'GHYX', 'ECY3',
    'target', 'ID2'
]
train = X_train.copy()
#Make info about true values in data of predictions
#
X_test = []
true_values = []
client_index = 0
for line in tqdm_notebook(test.values):

    client_index += 1

    info = line[:8]
    info_products = line[8:-1]
    indexes = [k for k, i in enumerate(info_products) if i == 1]

    X_test.append(list(info) + list(info_products) + [client_index])

    for true in test.columns[8:][indexes]:
        true_values.append(line[0] + ' X ' + true)

X_test = pd.DataFrame(X_test)
X_test.columns = [
    'ID', 'join_date', 'sex', 'marital_status', 'birth_year', 'branch_code',
    'occupation_code', 'occupation_category_code', 'P5DA', 'RIBP', '8NN1',
    '7POT', '66FJ', 'GYSR', 'SOP4', 'RVSZ', 'PYUQ', 'LJR9', 'N2MW', 'AHXO',
    'BSTQ', 'FM3X', 'K6QO', 'QBOL', 'JWFN', 'JZ9D', 'J9JW', 'GHYX', 'ECY3',
    'ID2'
]
test = X_test.copy()
train['marital_status'] = train['marital_status'].replace(['f'], ['F'])
df = train.append(test)<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>def create_date_featues(df):

    df['Join_Year'] = pd.to_datetime(df['join_date']).dt.year

    df['Join_Month'] = pd.to_datetime(df['join_date']).dt.month

    df['Join_Day'] = pd.to_datetime(df['join_date']).dt.day

    df['DayOfyear'] = pd.to_datetime(df['join_date']).dt.dayofyear

    return df
df = create_date_featues(df)
df['birth_year_bin'] = pd.cut(df['birth_year'], bins=5)
%%time
columns = [
    'P5DA', 'RIBP', '8NN1', '7POT', '66FJ', 'GYSR', 'SOP4', 'RVSZ', 'PYUQ',
    'LJR9', 'N2MW', 'AHXO', 'BSTQ', 'FM3X', 'K6QO', 'QBOL', 'JWFN', 'JZ9D',
    'J9JW', 'GHYX', 'ECY3'
]
for col in columns:
    df[col + '_' + 'sum'] = df.groupby('branch_code')[col].transform(sum)
for col in columns:
    df[col + '_' +
       'Join_year_sum'] = df.groupby('Join_Year')[col].transform(sum)
df['join_date'] = pd.to_datetime(df['join_date'])
for col in columns:
    df['from_arise_col_' +
       col] = (df['join_date'] -
               df.loc[df[col] == 1, 'join_date'].min()).dt.days
df['Number_of_Insurance_Bought'] = df.iloc[:, 8:29].sum(axis=1)


def mapper(df):
    if df['Number_of_Insurance_Bought'] == 1:
        return 'One'
    elif (df['Number_of_Insurance_Bought'] >
          1) & (df['Number_of_Insurance_Bought'] < 5):
        return 'Medium'
    elif (df['Number_of_Insurance_Bought'] >
          4) & (df['Number_of_Insurance_Bought'] < 8):
        return 'High'
    else:
        return 'Too High'


df['Insurance_Count'] = df.apply(lambda df: mapper(df), axis=1)
del df['Number_of_Insurance_Bought']
df['branch_start_year'] = df.groupby('branch_code')['Join_Year'].transform('min')
df['branch_since'] = 2020 - df['branch_start_year']
del df['branch_start_year']
df['Unique_customers_per_branch'] = df.groupby('branch_code')['ID'].transform('nunique')
df['Unique_Insurance_per_branch'] = df.groupby('branch_code')['target'].transform('nunique')

df['Unique_year_per_branch'] = df.groupby('branch_code')['Join_Year'].transform('nunique')
df['Unique_month_per_branch'] = df.groupby('branch_code')['Join_Month'].transform('nunique')
df['Unique_branch_per_year'] = df.groupby('Join_Year')['branch_code'].transform('nunique')
df['Age'] = df['Join_Year'] - df['birth_year']
df['Average_Age_per_branch'] = df.groupby('branch_code')['Age'].transform('mean')

df['Average_Age_per_occupation'] = df.groupby('occupation_code')['Age'].transform('mean')

for col in columns:
    df[col + '_' + 'meanAge'] = df.groupby(col)['Age'].transform('mean')

del df['Age']
df.reset_index(drop=True, inplace=True)
names_products = [
    'P5DA', 'RIBP', '8NN1', '7POT', '66FJ', 'GYSR', 'SOP4', 'RVSZ', 'PYUQ',
    'LJR9', 'N2MW', 'AHXO', 'BSTQ', 'FM3X', 'K6QO', 'QBOL', 'JWFN', 'JZ9D',
    'J9JW', 'GHYX', 'ECY3'
]
#Add glue togethered targets
#
for i, row in tqdm_notebook(df.iterrows()):
    res = []
    for c in names_products:
        if row[c] == 1:
            res.append(c)
    df.loc[df.index == i, 'product_comb'] = '_'.join(sorted(res))
df.head()<jupyter_output><empty_output><jupyter_text>Interaction Feature<jupyter_code>df['Join_Year'] = df['Join_Year'].astype(str)
df['birth_year_bin'] = df['birth_year_bin'].astype(str)
df['bc_oc'] = df['branch_code'] + '_' + df['occupation_code']
df['bc_occ'] = df['branch_code'] + '_' + df['occupation_category_code']<jupyter_output><empty_output><jupyter_text>Label Encoding<jupyter_code>from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
for col in [
        'product_comb', 'Insurance_Count', 'sex', 'marital_status',
        'branch_code', 'occupation_category_code', 'occupation_code'
]:
    df[col] = le.fit_transform(df[col])<jupyter_output><empty_output><jupyter_text>Frequency Encoding<jupyter_code>fe_pol = (df.groupby('product_comb').size()) / len(df)
df['product_comb_fe'] = df['product_comb'].apply(lambda x: fe_pol[x])
fe_pol = (df.groupby('bc_occ').size()) / len(df)
df['bc_occ'] = df['bc_occ'].apply(lambda x: fe_pol[x])
fe_pol = (df.groupby('bc_oc').size()) / len(df)
df['bc_oc'] = df['bc_oc'].apply(lambda x: fe_pol[x])
fe_pol = (df.groupby('birth_year_bin').size()) / len(df)
df['birth_year_bin'] = df['birth_year_bin'].apply(lambda x: fe_pol[x])
df['birth_year_bin'] = df['birth_year_bin'].astype(float)
fe_pol = (df.groupby('occupation_code').size()) / len(df)
df['occupation_code_fe'] = df['occupation_code'].apply(lambda x: fe_pol[x])
fe_pol = (df.groupby('occupation_category_code').size()) / len(df)
df['occupation_category_code'] = df['occupation_category_code'].apply(lambda x: fe_pol[x])
fe_pol = (df.groupby('sex').size()) / len(df)
df['sex_fe'] = df['sex'].apply(lambda x: fe_pol[x])
fe_pol = (df.groupby('Insurance_Count').size()) / len(df)
df['Insurance_Count_fe'] = df['Insurance_Count'].apply(lambda x: fe_pol[x])
df['Join_Year'] = df['Join_Year'].astype(float)
#Relationship between targets
#
for col in columns:
    for cols in columns:
        if col != cols:
            df[col + '_' + cols] = df.groupby(col)[cols].transform(sum)
df['num_freq'] = df.groupby('product_comb_fe')['ID'].transform('count')<jupyter_output><empty_output><jupyter_text>Getting back train and test<jupyter_code>train = df[:train.shape[0]]
test = df[-test.shape[0]:]
len(train),train['target'].nunique()<jupyter_output><empty_output><jupyter_text>Removing records if target count is less than 3<jupyter_code>train['target_count'] = train.groupby(['branch_code',
                                       'target'])['target'].transform('count')
train = train[train['target_count'] > 2]
del train['target_count']
len(train), train['target'].nunique()<jupyter_output><empty_output><jupyter_text>Label Encoding Target<jupyter_code>te = LabelEncoder()
train['target'] = te.fit_transform(train['target'])<jupyter_output><empty_output><jupyter_text>StratifiedKFold<jupyter_code>#LGB model
#
err = []
y_pred_tot_lgb = 0

fold = StratifiedKFold(n_splits=3, shuffle=True, random_state=1997)
i = 1
x = train.drop(columns={'join_date', 'ID', 'ID2', 'target'})
y = train[['target']]
ID = test['ID']
testing = test.drop(columns={'join_date', 'ID', 'ID2', 'target'})
for train_index, test_index in fold.split(x, y):
    x_train, x_val = x.iloc[train_index], x.iloc[test_index]
    y_train, y_val = y.iloc[train_index], y.iloc[test_index]
    m = LGBMClassifier(n_estimators=10000,
                       n_jobs=-1,
                       random_state=69,
                       learning_rate=0.01,
                       max_depth=5,
                       num_leaves=128,
                       colsample_bytree=0.5,
                       colsample_bynode=0.5,
                       min_data_in_leaf=40,
                       bagging_freq=2,
                       bagging_fraction=0.9,
                       reg_alpha=0.5,
                       reg_lambda=1)
    m.fit(x_train,
          y_train,
          eval_set=[(x_train, y_train), (x_val, y_val)],
          early_stopping_rounds=20,
          eval_metric='multi_logloss',
          verbose=200)
    pred_y = m.predict_proba(x_val)
    print(i, " err_lgm: ", log_loss(y_val, pred_y))
    err.append(log_loss(y_val, pred_y))
    pred_test = m.predict_proba(testing)
    y_pred_tot_lgb += pred_test
y_pred_tot_lgb = y_pred_tot_lgb / 3
(err[0] + err[1] + err[2]) / 3
#XGB model
#
err = []
y_pred_tot_xgb = 0

fold = StratifiedKFold(n_splits=3, shuffle=True, random_state=1997)
i = 1

for train_index, test_index in fold.split(x, y):
    x_train, x_val = x.iloc[train_index], x.iloc[test_index]
    y_train, y_val = y.iloc[train_index], y.iloc[test_index]
    m = XGBClassifier(
        n_estimators=10000,
        eta=0.1,
        n_jobs=-1,
        random_state=69,
        reg_alpha=0.5,  #reg_lambda=1.2 
        colsample_bytree=0.8,
        colsample_bylevel=0.8,
        colsample_bynode=0.8,
        subsample=0.9,
        gamma=1.5,
        max_depth=7)
    m.fit(x_train,
          y_train,
          eval_set=[(x_train, y_train), (x_val, y_val)],
          early_stopping_rounds=20,
          eval_metric='mlogloss',
          verbose=20)
    pred_y = m.predict_proba(x_val)
    print(i, " err_lgm: ", log_loss(y_val, pred_y))
    err.append(log_loss(y_val, pred_y))
    pred_test = m.predict_proba(testing)
    y_pred_tot_xgb += pred_test
y_pred_tot_xgb = y_pred_tot_xgb / 3
(err[0] + err[1] + err[2]) / 3<jupyter_output>[0]	validation_0-mlogloss:2.23928	validation_1-mlogloss:2.24216
Multiple eval metrics have been passed: 'validation_1-mlogloss' will be used for early stopping.

Will train until validation_1-mlogloss hasn't improved in 20 rounds.
[20]	validation_0-mlogloss:0.55627	validation_1-mlogloss:0.58586
[40]	validation_0-mlogloss:0.36262	validation_1-mlogloss:0.41120
[60]	validation_0-mlogloss:0.30980	validation_1-mlogloss:0.37351
[80]	validation_0-mlogloss:0.28610	validation_1-mlogloss:0.36247
[100]	validation_0-mlogloss:0.27054	validation_1-mlogloss:0.35819
[120]	validation_0-mlogloss:0.25927	validation_1-mlogloss:0.35662
[140]	validation_0-mlogloss:0.25005	validation_1-mlogloss:0.35568
[160]	validation_0-mlogloss:0.24265	validation_1-mlogloss:0.35481
[180]	validation_0-mlogloss:0.23612	validation_1-mlogloss:0.35453
[200]	validation_0-mlogloss:0.23081	validation_1-mlogloss:0.35423
[220]	validation_0-mlogloss:0.22614	validation_1-mlogloss:0.35409
[240]	validation_0-mlogloss:0.22208	validation_[...]<jupyter_text>Averaging submission<jupyter_code>pred = y_pred_tot_lgb * 0.6 + y_pred_tot_xgb * 0.4
y_test = pd.DataFrame(pred)
y_test.columns = te.inverse_transform(y_test.columns)
%%time
answer_mass = []
for i in range(test.shape[0]):
    test['ID'] = ID
    id = test['ID'].iloc[i]
    for c in y_test.columns:
        answer_mass.append([id + ' X ' + c, y_test[c].iloc[i]])

df_answer = pd.DataFrame(answer_mass)
df_answer.columns = ['ID X PCODE', 'Label']
for i in tqdm_notebook(range(df_answer.shape[0])):
    if df_answer['ID X PCODE'].iloc[i] in true_values:
        df_answer['Label'].iloc[i] = 1.0
df_answer.head()
sub1 = df_answer[['ID X PCODE', 'Label']]
sub1.reset_index(drop=True, inplace=True)
sub = pd.read_csv('SampleSubmission.csv')
sub.sort_values(by=['ID X PCODE'], inplace=True)
sub1.sort_values(by=['ID X PCODE'], inplace=True)
actual = sub1
findl = actual['ID X PCODE'].values
replacel = actual['Label'].values
sub.loc[sub['ID X PCODE'].isin(findl), ['Label']] = replacel
#Make submission
sub.to_csv('submiss.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16616552119129438, 'fraction_numerical': 0.02956758304696449, 'mean_word_length': 4.563122262046993, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '383093', 'license': 'MIT', 'n_tokens_mistral': 5635, 'n_tokens_neox': 5374, 'n_words': 870, 'path': 'Zimnat-lgb_best_score.ipynb', 'repo_name': 'dead-mazai/Zimnat_Insurance_top-12', 'size': '13968'}