<jupyter_start><jupyter_text>The process of developing a solution of the top 15%Pavel Nesterovhttp://pavelnesterov.info/<jupyter_code>%matplotlib inline
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
pd.set_option('display.max_columns', 500)
import seaborn as sns
sns.set_style("dark")
plt.rcParams['figure.figsize'] = 16, 12
from tqdm import tqdm, tqdm_notebook
import itertools as it
import pickle
import glob
import os
import string

from scipy import sparse

import nltk
import spacy

from sklearn.model_selection import StratifiedKFold, cross_val_score, GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import log_loss, make_scorer
from sklearn.decomposition import TruncatedSVD

from scipy.optimize import minimize

import eli5
from IPython.display import display

import xgboost as xgb
src = '/media/w/1c392724-ecf3-4615-8f3c-79368ec36380/DS Projects/Kaggle/Quora/data/'

df_train = pd.read_csv(src + 'train.csv', 
                       dtype={
                           'question1': np.str,
                           'question2': np.str
                       })
df_train['test_id'] = -1
df_test = pd.read_csv(src + 'test.csv', 
                      dtype={
                          'question1': np.str,
                          'question2': np.str
                      })
df_test['id'] = -1
df_test['qid1'] = -1
df_test['qid2'] = -1
df_test['is_duplicate'] = -1

df = pd.concat([df_train, df_test])
df['question1'] = df['question1'].fillna('')
df['question2'] = df['question2'].fillna('')
df['uid'] = np.arange(df.shape[0])
df = df.set_index(['uid'])
del(df_train, df_test)
ix_train = np.where(df['id'] >= 0)[0]
ix_test = np.where(df['id'] == -1)[0]
ix_is_dup = np.where(df['is_duplicate'] == 1)[0]
ix_not_dup = np.where(df['is_duplicate'] == 0)[0]
score0 = 6.01888
score1 = 28.52056
score05 = 0.69315
score025 = 0.47913
score075 = 1.19485
A = (score0 + score1)*np.log(0.5)/score05
eps = 10e-16
B = np.log(1 - eps)
C = np.log(eps)
r1 = (score1 - (C/B)*score0) / ((C*C/B) - B)
r0 = (-score1 - r1*B)/C

d = df[df['is_duplicate'] >= 0]['is_duplicate'].value_counts(normalize=True).to_dict()
gamma_0 = r0/d[0]
gamma_1 = r1/d[1]
def link_function(x):
    return gamma_1*x/(gamma_1*x + gamma_0*(1 - x))

support = np.linspace(0, 1, 1000)
values = link_function(support)

fig, ax = plt.subplots()
ax.plot(support, values)
ax.set_title('Link transformation', fontsize=20)
ax.set_xlabel('x')
ax.set_ylabel('f(x)')
plt.show()
df['len1'] = df['question1'].str.len().astype(np.float32)
df['len2'] = df['question2'].str.len().astype(np.float32)
df['abs_diff_len1_len2'] = np.abs(df['len1'] - df['len2'])
def plot_real_feature(fname):
    fig = plt.figure()
    ax1 = plt.subplot2grid((3, 2), (0, 0), colspan=2)
    ax2 = plt.subplot2grid((3, 2), (1, 0), colspan=2)
    ax3 = plt.subplot2grid((3, 2), (2, 0))
    ax4 = plt.subplot2grid((3, 2), (2, 1))
    ax1.set_title('Distribution of %s' % fname, fontsize=20)
    sns.distplot(df.loc[ix_train][fname], 
                 bins=50, 
                 ax=ax1)    
    sns.distplot(df.loc[ix_is_dup][fname], 
                 bins=50, 
                 ax=ax2,
                 label='is dup')    
    sns.distplot(df.loc[ix_not_dup][fname], 
                 bins=50, 
                 ax=ax2,
                 label='not dup')
    ax2.legend(loc='upper right', prop={'size': 18})
    sns.boxplot(y=fname, 
                x='is_duplicate', 
                data=df.loc[ix_train], 
                ax=ax3)
    sns.violinplot(y=fname, 
                   x='is_duplicate', 
                   data=df.loc[ix_train], 
                   ax=ax4)
    plt.show()
    
plot_real_feature('abs_diff_len1_len2')
max_in_dup = df.loc[ix_is_dup]['abs_diff_len1_len2'].max()
# print'Maximum among duplicates:       ', max_in_dup
max_in_not_dups = df.loc[ix_not_dup]['abs_diff_len1_len2'].max()
# print'Maximum among non-duplicates:     ', max_in_not_dups
# print'Maximum among non-duplicates: ', (df.loc[ix_train]['abs_diff_len1_len2'] > max_in_dup).sum()
std_in_dups = df.loc[ix_is_dup]['abs_diff_len1_len2'].std()
# print'Standard deviation in duplicates:', std_in_dups
replace_value = max_in_dup + 2*std_in_dups
# print'New value:              ', replace_value
df['abs_diff_len1_len2'] = df['abs_diff_len1_len2'].apply(lambda x: x if x < replace_value else replace_value)
plot_real_feature('abs_diff_len1_len2')
df['log_abs_diff_len1_len2'] = np.log(df['abs_diff_len1_len2'] + 1)
plot_real_feature('log_abs_diff_len1_len2')
df['ratio_len1_len2'] = df['len1'].apply(lambda x: x if x > 0.0 else 1.0)/\
                        df['len2'].apply(lambda x: x if x > 0.0 else 1.0)
plot_real_feature('ratio_len1_len2')
max_in_dup = df.loc[ix_is_dup]['ratio_len1_len2'].max()
# print'Maximum among duplicates:        ', max_in_dup
max_in_not_dups = df.loc[ix_not_dup]['ratio_len1_len2'].max()
# print'Maximum among non-duplicates:      ', max_in_not_dups
# print'Number of lines greater than threshold: ', (df.loc[ix_train]['ratio_len1_len2'] > max_in_dup).sum()
std_in_dups = df.loc[ix_is_dup]['ratio_len1_len2'].std()
# print'Number of lines greater than threshold: ', std_in_dups
replace_value = max_in_dup + 2*std_in_dups
# print'New value:               ', replace_value
df['ratio_len1_len2'] = df['ratio_len1_len2'].apply(lambda x: x if x < replace_value else replace_value)
plot_real_feature('ratio_len1_len2')
df['log_ratio_len1_len2'] = np.log(df['ratio_len1_len2'] + 1)
plot_real_feature('log_ratio_len1_len2')
def plot_corr(predictors):
    predictors = predictors[:]
    predictors += ['is_duplicate']
    mcorr = df.loc[ix_train][predictors].corr()
    mask = np.zeros_like(mcorr, dtype=np.bool)
    mask[np.triu_indices_from(mask)] = True
    cmap = sns.diverging_palette(220, 10, as_cmap=True)
    g = sns.heatmap(mcorr, mask=mask, cmap=cmap, square=True, annot=True, fmt='0.2f')
    g.set_xticklabels(predictors, rotation=90)
    g.set_yticklabels(reversed(predictors))
    plt.show()
    
plot_corr(df.columns[7:].tolist())
%%time
if os.path.isfile('./../tmp/cv_char.pkl') and os.path.isfile('./../tmp/ch_freq.pkl'):
    with open('./../tmp/cv_char.pkl', 'rb') as f:
        cv_char = pickle.load(f)
    with open('./../tmp/ch_freq.pkl', 'rb') as f:
        ch_freq = pickle.load(f)
else:
    cv_char = CountVectorizer(ngram_range=(1, 3), analyzer='char')
    ch_freq = np.array(cv_char.fit_transform(df['question1'].tolist() + df['question2'].tolist()).sum(axis=0))[0, :]
    with open('./../tmp/cv_char.pkl', 'wb') as f:
        pickle.dump(cv_char, f)
    with open('./../tmp/ch_freq.pkl', 'wb') as f:
        pickle.dump(ch_freq, f)
unigrams = dict([(k, v) for (k, v) in cv_char.vocabulary_.items() if len(k) == 1])
ix_unigrams = sorted(unigrams.values())
print('Unigrams:', len(unigrams))
bigrams = dict([(k, v) for (k, v) in cv_char.vocabulary_.items() if len(k) == 2])
ix_bigrams = sorted(bigrams.values())
trigrams = dict([(k, v) for (k, v) in cv_char.vocabulary_.items() if len(k) == 3])
ix_trigrams = sorted(trigrams.values())
print('Trigrams:', len(trigrams))
%%time
def save_sparse_csr(fname, sm):
    np.savez(fname, 
             data=sm.data, 
             indices=sm.indices,
             indptr=sm.indptr, 
             shape=sm.shape)

def load_sparse_csr(fname):
    loader = np.load(fname)
    return sparse.csr_matrix((
        loader['data'], 
        loader['indices'], 
        loader['indptr']),
        shape=loader['shape'])

if os.path.isfile('./../tmp/m_q1.npz') and os.path.isfile('./../tmp/m_q2.npz'):
    m_q1 = load_sparse_csr('./../tmp/m_q1.npz')
    m_q2 = load_sparse_csr('./../tmp/m_q2.npz')
else:
    m_q1 = cv_char.transform(df['question1'].values)
    m_q2 = cv_char.transform(df['question2'].values)
    save_sparse_csr('./../tmp/m_q1.npz', m_q1)
    save_sparse_csr('./../tmp/m_q2.npz', m_q2)
v_num = (m_q1[:, ix_unigrams] > 0).minimum((m_q2[:, ix_unigrams] > 0)).sum(axis=1)
v_den = (m_q1[:, ix_unigrams] > 0).maximum((m_q2[:, ix_unigrams] > 0)).sum(axis=1)
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['unigram_jaccard'] = v_score
# We take into account each letter more than once
v_num = m_q1[:, ix_unigrams].minimum(m_q2[:, ix_unigrams]).sum(axis=1)
v_den = m_q1[:, ix_unigrams].sum(axis=1) + m_q2[:, ix_unigrams].sum(axis=1)
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['unigram_all_jaccard'] = v_score
# We take into account each letter more than once
# Normalize the maximum value, and not the sum
v_num = m_q1[:, ix_unigrams].minimum(m_q2[:, ix_unigrams]).sum(axis=1)
v_den = m_q1[:, ix_unigrams].maximum(m_q2[:, ix_unigrams]).sum(axis=1)
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['unigram_all_jaccard_max'] = v_score
v_num = (m_q1[:, ix_bigrams] > 0).minimum((m_q2[:, ix_bigrams] > 0)).sum(axis=1)
v_den = (m_q1[:, ix_bigrams] > 0).maximum((m_q2[:, ix_bigrams] > 0)).sum(axis=1)
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['bigram_jaccard'] = v_score
df.loc[ix_train].groupby(['is_duplicate'])['bigram_jaccard'].agg(
    {
        'min': np.min,
        'max': np.max,
        'p1': lambda x: np.percentile(x, q=0.01),
        'p99': lambda x: np.percentile(x, q=99.99)
    }
)
df.loc[df['bigram_jaccard'] < -1.478751, 'bigram_jaccard'] = -1.478751
df.loc[df['bigram_jaccard'] > 1.0, 'bigram_jaccard'] = 1.0
# We take into account each letter more than once
v_num = m_q1[:, ix_bigrams].minimum(m_q2[:, ix_bigrams]).sum(axis=1)
v_den = m_q1[:, ix_bigrams].sum(axis=1) + m_q2[:, ix_bigrams].sum(axis=1)
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['bigram_all_jaccard'] = v_score
# We take into account each letter more than once
# Normalize the maximum value, and not the sum
v_num = m_q1[:, ix_bigrams].minimum(m_q2[:, ix_bigrams]).sum(axis=1)
v_den = m_q1[:, ix_bigrams].maximum(m_q2[:, ix_bigrams]).sum(axis=1)
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['bigram_all_jaccard_max'] = v_score
m_q1 = m_q1[:, ix_trigrams]
m_q2 = m_q2[:, ix_trigrams]
v_num = (m_q1 > 0).minimum((m_q2 > 0)).sum(axis=1)
v_den = (m_q1 > 0).maximum((m_q2 > 0)).sum(axis=1)
v_den[np.where(v_den == 0)] = 1
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['trigram_jaccard'] = v_score
# учитываем каждую букву больше одного раза
v_num = m_q1.minimum(m_q2).sum(axis=1)
v_den = m_q1.sum(axis=1) + m_q2.sum(axis=1)
v_den[np.where(v_den == 0)] = 1
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['trigram_all_jaccard'] = v_score
# We take into account each letter more than once
# Normalize the maximum value, and not the sum
v_num = m_q1.minimum(m_q2).sum(axis=1)
v_den = m_q1.maximum(m_q2).sum(axis=1)
v_den[np.where(v_den == 0)] = 1
v_score = np.array(v_num.flatten()).astype(np.float32)[0, :]/np.array(v_den.flatten())[0, :]

df['trigram_all_jaccard_max'] = v_score<jupyter_output><empty_output><jupyter_text>Tf-idf and pair metrics on trigrams<jupyter_code>tft = TfidfTransformer(
    norm='l2', 
    use_idf=True, 
    smooth_idf=True, 
    sublinear_tf=False)

tft = tft.fit(sparse.vstack((m_q1, m_q2)))
m_q1_tf = tft.transform(m_q1)
m_q2_tf = tft.transform(m_q2)

v_num = np.array(m_q1_tf.multiply(m_q2_tf).sum(axis=1))[:, 0]
v_den = np.array(np.sqrt(m_q1_tf.multiply(m_q1_tf).sum(axis=1)))[:, 0] * \
        np.array(np.sqrt(m_q2_tf.multiply(m_q2_tf).sum(axis=1)))[:, 0]
v_num[np.where(v_den == 0)] = 1
v_den[np.where(v_den == 0)] = 1

v_score = 1 - v_num/v_den

df['trigram_tfidf_cosine'] = v_score
tft = TfidfTransformer(
    norm='l2', 
    use_idf=True, 
    smooth_idf=True, 
    sublinear_tf=False)

tft = tft.fit(sparse.vstack((m_q1, m_q2)))
m_q1_tf = tft.transform(m_q1)
m_q2_tf = tft.transform(m_q2)

v_score = (m_q1_tf - m_q2_tf)
v_score = np.sqrt(np.array(v_score.multiply(v_score).sum(axis=1))[:, 0])

df['trigram_tfidf_l2_euclidean'] = v_score
tft = TfidfTransformer(
    norm='l1', 
    use_idf=True, 
    smooth_idf=True, 
    sublinear_tf=False)

tft = tft.fit(sparse.vstack((m_q1, m_q2)))
m_q1_tf = tft.transform(m_q1)
m_q2_tf = tft.transform(m_q2)

v_score = (m_q1_tf - m_q2_tf)
v_score = np.sqrt(np.array(v_score.multiply(v_score).sum(axis=1))[:, 0])

df['trigram_tfidf_l1_euclidean'] = v_score
tft = TfidfTransformer(
    norm='l2', 
    use_idf=False, 
    smooth_idf=True, 
    sublinear_tf=False)

tft = tft.fit(sparse.vstack((m_q1, m_q2)))
m_q1_tf = tft.transform(m_q1)
m_q2_tf = tft.transform(m_q2)

v_score = (m_q1_tf - m_q2_tf)
v_score = np.sqrt(np.array(v_score.multiply(v_score).sum(axis=1))[:, 0])

df['trigram_tf_l2_euclidean'] = v_score
tmp = set(ix_trigrams)
tmp = [k for (k, v) in sorted(cv_char.vocabulary_.items(), key=lambda t: t[1]) if v in tmp]
tmp = tmp + tmp
df[:1000][df['question1'][:1000].apply(lambda s: 'o v' in s)]['question1'].tolist()
df[:1000][df['question1'][:1000].apply(lambda s: 'o v' in s)]['question2'].tolist()
df[:1000][df['question1'][:1000].apply(lambda s: '.co' in s)]['question1'].tolist()
df[:1000][df['question1'][:1000].apply(lambda s: '.co' in s)]['question2'].tolist()
if not os.path.isfile('./../tmp/1_svd.pkl'):
    svd = TruncatedSVD(n_components=100)
    m_svd = svd.fit_transform(sparse.csc_matrix(sparse.hstack((m_q1_tf, m_q2_tf))))
    with open('./../tmp/1_svd.pkl', 'wb') as f:
        pickle.dump(svd, f)
    with open('./../tmp/1_m_svd.npz', 'wb') as f:
        np.savez(f, m_svd)
else:
    with open('./../tmp/1_svd.pkl', 'rb') as f:
        svd = pickle.load(f)
    with open('./../tmp/1_m_svd.npz', 'rb') as f:
        m_svd = np.load(f)['arr_0']
df['m_q1_q2_tf_svd0'] = m_svd[:, 0]
df['m_q1_q2_tf_svd1'] = m_svd[:, 0]
if not os.path.isfile('./../tmp/2_svd.pkl'):
    svd = TruncatedSVD(n_components=100)
    m_svd = svd.fit_transform(sparse.csc_matrix(sparse.vstack((m_q1_tf, m_q2_tf))))
    with open('./../tmp/2_svd.pkl', 'wb') as f:
        pickle.dump(svd, f)
    with open('./../tmp/2_m_svd.npz', 'wb') as f:
        np.savez(f, m_svd)
else:
    with open('./../tmp/2_svd.pkl', 'rb') as f:
        svd = pickle.load(f)
    with open('./../tmp/2_m_svd.npz', 'rb') as f:
        m_svd = np.load(f)['arr_0']
m_svd_q1 = m_svd[:int(m_svd.shape[0]/2), :]
m_svd_q2 = m_svd[int(m_svd.shape[0]/2):, :]
del(m_svd)
df['m_vstack_svd_q1_q1_euclidean'] = ((m_svd_q1 - m_svd_q2)**2).mean(axis=1)
num = (m_svd_q1*m_svd_q2).sum(axis=1)
den = np.sqrt((m_svd_q1**2).sum(axis=1))*np.sqrt((m_svd_q2**2).sum(axis=1))
num[np.where(den == 0)] = 0
den[np.where(den == 0)] = 1
df['m_vstack_svd_q1_q1_cosine'] = 1 - num/den
m_svd = m_svd_q1*m_svd_q2
m_svd = np.abs(m_svd_q1 - m_svd_q2)
import en_core_web_md

nlp = en_core_web_md.load()
df.head()['question1'].apply(lambda s: ' '.join([c.lemma_ for c in nlp(str(s)) if c.lemma_  != '?']))
SYMBOLS = set(' '.join(string.punctuation).split(' ') + ['...', '“', '”', '\'ve'])

if not os.path.isfile('./../tmp/bow_lemma.pkl'):
    q1 = []

    for doc in nlp.pipe(df['question1'], n_threads=8, batch_size=10000):
        q1.append([c.lemma_ for c in doc if c.lemma_ not in SYMBOLS])

    q2 = []

    for doc in nlp.pipe(df['question2'], n_threads=8, batch_size=10000):
        q2.append([c.lemma_ for c in doc if c.lemma_ not in SYMBOLS])
        
    with open('./../tmp/bow_lemma.pkl', 'wb') as f:
        pickle.dump({
            'q1': q1,
            'q2': q2
        }, f)
else:
    with open('./../tmp/bow_lemma.pkl', 'rb') as f:
        tmp = pickle.load(f)
        q1 = tmp['q1']
        q2 = tmp['q2']
        del(tmp)
%%time
if os.path.isfile('./../tmp/cv_word_lemma.pkl') and os.path.isfile('./../tmp/wl_freq.pkl'):
    with open('./../tmp/cv_word_lemma.pkl', 'rb') as f:
        cv_words = pickle.load(f)
    with open('./../tmp/wl_freq.pkl', 'rb') as f:
        w_freq = pickle.load(f)
else:
    cv_words = CountVectorizer(ngram_range=(1, 1), analyzer='word')
    w_freq = np.array(cv_words.fit_transform(
        [' '.join(s) for s in q1] + [' '.join(s) for s in q2]).sum(axis=0))[0, :]
    with open('./../tmp/cv_word_lemma.pkl', 'wb') as f:
        pickle.dump(cv_words, f)
    with open('./../tmp/wl_freq.pkl', 'wb') as f:
        pickle.dump(w_freq, f)
if os.path.isfile('./../tmp/m_q1_wl.npz') and os.path.isfile('./../tmp/m_q2_wl.npz'):
    m_q1 = load_sparse_csr('./../tmp/m_q1_wl.npz')
    m_q2 = load_sparse_csr('./../tmp/m_q2_wl.npz')
else:
    m_q1 = cv_words.transform([' '.join(s) for s in q1])
    m_q2 = cv_words.transform([' '.join(s) for s in q2])
    save_sparse_csr('./../tmp/m_q1_wl.npz', m_q1)
    save_sparse_csr('./../tmp/m_q2_wl.npz', m_q2)
tft = TfidfTransformer(
    norm='l2', 
    use_idf=True, 
    smooth_idf=True, 
    sublinear_tf=False)

tft = tft.fit(sparse.vstack((m_q1, m_q2)))
m_q1_tf = tft.transform(m_q1)
m_q2_tf = tft.transform(m_q2)

v_num = np.array(m_q1_tf.multiply(m_q2_tf).sum(axis=1))[:, 0]
v_den = np.array(np.sqrt(m_q1_tf.multiply(m_q1_tf).sum(axis=1)))[:, 0] * \
        np.array(np.sqrt(m_q2_tf.multiply(m_q2_tf).sum(axis=1)))[:, 0]
v_num[np.where(v_den == 0)] = 1
v_den[np.where(v_den == 0)] = 1

v_score = 1 - v_num/v_den

df['1wl_tfidf_cosine'] = v_score
plot_real_feature('1wl_tfidf_cosine')
tft = TfidfTransformer(
    norm='l2', 
    use_idf=True, 
    smooth_idf=True, 
    sublinear_tf=False)

tft = tft.fit(sparse.vstack((m_q1, m_q2)))
m_q1_tf = tft.transform(m_q1)
m_q2_tf = tft.transform(m_q2)

v_score = (m_q1_tf - m_q2_tf)
v_score = np.sqrt(np.array(v_score.multiply(v_score).sum(axis=1))[:, 0])

df['1wl_tfidf_l2_euclidean'] = v_score
plot_real_feature('1wl_tfidf_l2_euclidean')
tft = TfidfTransformer(
    norm='l2', 
    use_idf=False, 
    smooth_idf=True, 
    sublinear_tf=False)

tft = tft.fit(sparse.vstack((m_q1, m_q2)))
m_q1_tf = tft.transform(m_q1)
m_q2_tf = tft.transform(m_q2)

v_score = (m_q1_tf - m_q2_tf)
v_score = np.sqrt(np.array(v_score.multiply(v_score).sum(axis=1))[:, 0])

df['1wl_tf_l2_euclidean'] = v_score
plot_real_feature('1wl_tf_l2_euclidean')
del(m_q1, m_q2)
df_train = df[:df_train.shape[0]]
df_test = df[df_train.shape[0]:]
df_train.to_csv('train_mephistopeheles_features.csv', index = False)
df_test.to_csv('test_mephistopeheles_features.csv', index = False)
xgb_models = []
if not os.path.isfile('25_pred_fixed.csv'):
    def log_loss_lf_xgb(y_pred, y_true):
        return 'llf', log_loss(y_true.get_label(), link_function(y_pred), eps=eps)

    xgb_params = {
        'max_depth': 9, 
        'learning_rate': 0.1,
        'n_estimators': 2500, 
        'objective': 'binary:logistic',
        'nthread': 16, 
        'gamma': 0, 
        'subsample': 0.75, 
        'colsample_bytree': 0.75, 
        'colsample_bylevel': 1,
        'reg_alpha': 0, 
        'reg_lambda': 1, 
        'scale_pos_weight': 1
    }

    y_test = []
    folder = QuoraKFold(n_splits=10, r_split=0.7)
    splits = folder.split(np.zeros(ix_train.shape[0]), df.loc[ix_train, 'is_duplicate'])
    for ix_first, ix_second in tqdm_notebook(splits, total=10):

        model = xgb.XGBClassifier(silent=True).set_params(**xgb_params)
        model = model.fit(df.loc[ix_first, predictors], df.loc[ix_first, 'is_duplicate'], 
                          eval_set=[(df.loc[ix_second, predictors], df.loc[ix_second, 'is_duplicate'])], 
                          eval_metric=log_loss_lf_xgb,
                          early_stopping_rounds=100, 
                          verbose=False)
        y_test.append(model.predict_proba(df.loc[ix_test, predictors])[:, 1])
        xgb_models.append(model)

    y_test_pred = np.array(y_test).T.mean(axis=1)
    y_test_pred_fixed = link_function(y_test_pred)
    
    pd.DataFrame.from_records(
        zip(df.loc[ix_test]['test_id'].values, 
            y_test_pred_fixed), 
        columns=['test_id', 'is_duplicate']).to_csv('./../submits/25_pred_fixed.csv', index=False)
if not os.path.isfile('./../tmp/bm1.pkl'):
    with open('./../tmp/bm1.pkl', 'wb') as f:
        pickle.dump(xgb_models, f)
else:
    with open('./../tmp/bm1.pkl', 'rb') as f:
        xgb_models = pickle.load(f)
model = xgb_models[0]
display(
    eli5.explain_weights_xgboost(
        model, 
        feature_names=predictors.tolist(), 
        top=len(predictors)))
xgb.plot_importance(model)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18232982388045219, 'fraction_numerical': 0.03575760516229198, 'mean_word_length': 3.7547866205305653, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '186968', 'license': 'MIT', 'n_tokens_mistral': 9303, 'n_tokens_neox': 8899, 'n_words': 1376, 'path': 'features/.ipynb_checkpoints/04.05 - Mephistopheles Features-Copy1-checkpoint.ipynb', 'repo_name': 'susantamoh84/Kaggle-Quora', 'size': '20611'}