<jupyter_start><jupyter_text>- V1 : LGBM STACKING - V2 : LGBM, MLP16 STACKING- V3 : V2 + pred 5 score 3<jupyter_code>import warnings
warnings.filterwarnings('ignore')

import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm_notebook

from sklearn import svm, neighbors, linear_model, neural_network

from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import StandardScaler, MinMaxScaler, QuantileTransformer
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import VarianceThreshold
from sklearn.decomposition import PCA, TruncatedSVD, KernelPCA

from sklearn.mixture import GaussianMixture as GMM
from sklearn.metrics import silhouette_score


import lightgbm as lgb
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn import preprocessing
from sklearn import svm, neighbors, linear_model
import gc
warnings.filterwarnings('ignore')


from sklearn.neural_network import MLPClassifier
from sklearn.naive_bayes import GaussianNB, BernoulliNB
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import Matern, RationalQuadratic
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.decomposition import FastICA, TruncatedSVD, PCA
from sklearn.ensemble import RandomForestClassifier
import lightgbm as lgb
import xgboost as xgb
import catboost as cat

from tqdm import *
%%time
train_df = pd.read_csv('../input/train.csv')
test_df = pd.read_csv('../input/test.csv')
train_columns = [c for c in train_df.columns if c not in ['id','target','wheezy-copper-turtle-magic']]

magic_variance_over2 = {}
for magic in sorted(train_df['wheezy-copper-turtle-magic'].unique()):
    temp = train_df.loc[train_df['wheezy-copper-turtle-magic']==magic]
    std = temp[train_columns].std()
    magic_variance_over2[magic] = list(std.index.values[np.where(std >2)])
class hist_model(object):
    
    def __init__(self, bins=50):
        self.bins = bins
        
    def fit(self, X):
        
        bin_hight, bin_edge = [], []
        
        for var in X.T:
            # get bins hight and interval
            bh, bedge = np.histogram(var, bins=self.bins)
            bin_hight.append(bh)
            bin_edge.append(bedge)
        
        self.bin_hight = np.array(bin_hight)
        self.bin_edge = np.array(bin_edge)
   

    def predict(self, X):
        
        scores = []
        for obs in X:
            obs_score = []
            for i, var in enumerate(obs):
                # find wich bin obs is in
                bin_num = (var > self.bin_edge[i]).argmin()-1
                obs_score.append(self.bin_hight[i, bin_num]) # find bin hitght
            
            scores.append(np.mean(obs_score))
        
        return np.array(scores)
from sklearn.neighbors import KNeighborsClassifier, RadiusNeighborsClassifier
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Lasso, LassoLars
random_state = 42
debug = True
debug = False
svnu_params = {'probability':True, 'kernel':'poly','degree':4,'gamma':'auto','nu':0.4,'coef0':0.08, 'random_state':4}
svnu2_params = {'probability':True, 'kernel':'poly','degree':2,'gamma':'auto','nu':0.4,'coef0':0.08, 'random_state':4}
svc_params = {'probability':True,'kernel':'poly','degree':4,'gamma':'auto', 'random_state':4}
lr_params = {'solver':'liblinear','penalty':'l1','C':0.05,'n_jobs':-1, 'random_state':42}
mlp16_params = {'activation':'relu','solver':'lbfgs','tol':1e-06, 'hidden_layer_sizes':(16, ), 'random_state':42}
mlp128_params = {'activation':'relu','solver':'lbfgs','tol':1e-06, 'hidden_layer_sizes':(128, ), 'random_state':42}
def get_oofs(random_state):
    oof_nusvc = np.zeros(len(train_df))
    preds_nusvc = np.zeros(len(test_df))

    oof_nusvc2 = np.zeros(len(train_df))
    preds_nusvc2 = np.zeros(len(test_df))

    oof_qda = np.zeros(len(train_df))
    preds_qda = np.zeros(len(test_df))

    oof_svc = np.zeros(len(train_df))
    preds_svc = np.zeros(len(test_df))
    
    oof_knn = np.zeros(len(train_df))
    preds_knn = np.zeros(len(test_df))
    
    oof_lr = np.zeros(len(train_df))
    preds_lr = np.zeros(len(test_df))
    
    cols = [c for c in train_df.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]

    for i in tqdm_notebook(range(512)):

        # each magic
        train = train_df[train_df['wheezy-copper-turtle-magic'] == i]
        test = test_df[test_df['wheezy-copper-turtle-magic'] == i]

        # for oof
        train_idx_origin = train.index
        test_idx_origin = test.index


        # start point

        # new cols
        cols = magic_variance_over2[i]

        X_train = train.reset_index(drop=True)[cols].values
        y_train = train.reset_index(drop=True).target

        X_test = test[cols].values

        # vstack
        data = np.vstack([X_train, X_test])
        
        # PCA
        data = KernelPCA(n_components=len(cols), kernel='cosine', random_state=random_state).fit_transform(data)
        
        # Bad
        '''
        gmm_pred = np.zeros((len(data), 5))
        for j in range(5):
            gmm = GMM(n_components=4, random_state=random_state + j, max_iter=1000).fit(data)
            gmm_pred[:, j] += gmm.predict(data)
        '''
          
        # original
        gmm = GMM(n_components=5, random_state=random_state, max_iter=1000).fit(data)
        gmm_pred = gmm.predict_proba(data)
        gmm_score = gmm.score_samples(data)
        gmm_label = gmm.predict(data)
        
        hist = hist_model(); hist.fit(data)
        hist_pred = hist.predict(data).reshape(-1, 1)

        data = np.hstack([data, gmm_pred])

        # HOXI
        data = np.hstack([data, gmm_pred])
        data = np.hstack([data, gmm_pred])
        data = np.hstack([data, gmm_pred])
        
        # Add Some Features
        data = np.hstack([data, gmm_pred])
        data = np.hstack([data, hist_pred, gmm_score.reshape(-1, 1)])
        data = np.hstack([data, gmm_score.reshape(-1, 1)])
        data = np.hstack([data, gmm_score.reshape(-1, 1)])

        # STANDARD SCALER
        data = StandardScaler().fit_transform(data)

        # new train/test
        X_train = data[:X_train.shape[0]]
        X_test = data[X_train.shape[0]:]

        fold = StratifiedKFold(n_splits=5, random_state=random_state)
        for tr_idx, val_idx in fold.split(X_train, gmm_label[:X_train.shape[0]]):
            
            # NuSVC 1
            clf = svm.NuSVC(**svnu_params)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_nusvc[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_nusvc[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits

            # NuSVC 2
            clf = svm.NuSVC(**svnu2_params)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_nusvc2[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_nusvc2[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits


            # qda 3
            clf = QuadraticDiscriminantAnalysis(reg_param=0.111)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_qda[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_qda[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits

            # SVC 4
            clf = svm.SVC(**svc_params)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_svc[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_svc[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
            
            # knn 8
            clf = KNeighborsClassifier(n_neighbors=16)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_knn[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_knn[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits   
            
            # LR 5
            clf = linear_model.LogisticRegression(**lr_params)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_lr[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_lr[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
            
    oof_train = pd.DataFrame()

    oof_train['nusvc'] = oof_nusvc
    oof_train['nusvc2'] = oof_nusvc2
    oof_train['qda'] = oof_qda
    oof_train['svc'] = oof_svc
    oof_train['knn'] = oof_knn
    oof_train['lr'] = oof_lr
    
    oof_test = pd.DataFrame()

    oof_test['nusvc'] = preds_nusvc
    oof_test['nusvc2'] = preds_nusvc2
    oof_test['qda'] = preds_qda
    oof_test['svc'] = preds_svc
    oof_test['knn'] = preds_knn
    oof_test['lr'] = preds_lr

    print('nusvc', roc_auc_score(train_df['target'], oof_nusvc))
    print('nusvc2', roc_auc_score(train_df['target'], oof_nusvc2))
    print('qda', roc_auc_score(train_df['target'], oof_qda))
    print('svc', roc_auc_score(train_df['target'], oof_svc))
    print('knn', roc_auc_score(train_df['target'], oof_knn))
    print('knn', roc_auc_score(train_df['target'], oof_lr))
    
    return oof_train, oof_test
def get_oofs_2(random_state):
    oof_nusvc = np.zeros(len(train_df))
    preds_nusvc = np.zeros(len(test_df))

    oof_nusvc2 = np.zeros(len(train_df))
    preds_nusvc2 = np.zeros(len(test_df))

    oof_qda = np.zeros(len(train_df))
    preds_qda = np.zeros(len(test_df))

    oof_svc = np.zeros(len(train_df))
    preds_svc = np.zeros(len(test_df))
    
    oof_knn = np.zeros(len(train_df))
    preds_knn = np.zeros(len(test_df))

    oof_lr = np.zeros(len(train_df))
    preds_lr = np.zeros(len(test_df))
    
    cols = [c for c in train_df.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]

    for i in tqdm_notebook(range(512)):

        # each magic
        train = train_df[train_df['wheezy-copper-turtle-magic'] == i]
        test = test_df[test_df['wheezy-copper-turtle-magic'] == i]

        # for oof
        train_idx_origin = train.index
        test_idx_origin = test.index


        # start point

        # new cols
        cols = magic_variance_over2[i]

        X_train = train.reset_index(drop=True)[cols].values
        y_train = train.reset_index(drop=True).target

        X_test = test[cols].values

        # vstack
        data = np.vstack([X_train, X_test])

        # PCA
        data = KernelPCA(n_components=len(cols), kernel='cosine', random_state=random_state).fit_transform(data)

        # Bad
        '''
        gmm_pred = np.zeros((len(data), 5))
        for j in range(5):
            gmm = GMM(n_components=4, random_state=random_state + j, max_iter=1000).fit(data)
            gmm_pred[:, j] += gmm.predict(data)
        '''
            
        # original
        gmm = GMM(n_components=5, random_state=random_state, max_iter=1000, init_params='random').fit(data)
        gmm_pred = gmm.predict_proba(data)
        gmm_score = gmm.score_samples(data)
        gmm_label = gmm.predict(data)
        
        hist = hist_model(); hist.fit(data)
        hist_pred = hist.predict(data).reshape(-1, 1)

        data = np.hstack([data, gmm_pred])
        
        # HOXI
        data = np.hstack([data, gmm_pred])
        data = np.hstack([data, gmm_pred])
        data = np.hstack([data, gmm_pred])
        
        # Add Some Features
        data = np.hstack([data, gmm_pred])
        data = np.hstack([data, hist_pred, gmm_score.reshape(-1, 1)])
        data = np.hstack([data, gmm_score.reshape(-1, 1)])
        data = np.hstack([data, gmm_score.reshape(-1, 1)])

        # STANDARD SCALER
        data = StandardScaler().fit_transform(data)

        # new train/test
        X_train = data[:X_train.shape[0]]
        X_test = data[X_train.shape[0]:]

        fold = StratifiedKFold(n_splits=5, random_state=random_state)
        for tr_idx, val_idx in fold.split(X_train, gmm_label[:X_train.shape[0]]):
            
            # NuSVC 1
            clf = svm.NuSVC(**svnu_params)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_nusvc[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_nusvc[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits

            # NuSVC 2
            clf = svm.NuSVC(**svnu2_params)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_nusvc2[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_nusvc2[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits


            # qda 3
            clf = QuadraticDiscriminantAnalysis(reg_param=0.111)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_qda[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_qda[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits

            # SVC 4
            clf = svm.SVC(**svc_params)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_svc[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_svc[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
            
            # knn 8
            clf = KNeighborsClassifier(n_neighbors=16)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_knn[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_knn[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits   
            
            # LR 5
            clf = linear_model.LogisticRegression(**lr_params)
            clf.fit(X_train[tr_idx], y_train[tr_idx])
            oof_lr[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
            preds_lr[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
            
    oof_train = pd.DataFrame()

    oof_train['nusvc'] = oof_nusvc
    oof_train['nusvc2'] = oof_nusvc2
    oof_train['qda'] = oof_qda
    oof_train['svc'] = oof_svc
    oof_train['knn'] = oof_knn
    oof_train['lr'] = oof_lr
    
    oof_test = pd.DataFrame()

    oof_test['nusvc'] = preds_nusvc
    oof_test['nusvc2'] = preds_nusvc2
    oof_test['qda'] = preds_qda
    oof_test['svc'] = preds_svc
    oof_test['knn'] = preds_knn
    oof_test['lr'] = preds_lr

    print('nusvc', roc_auc_score(train_df['target'], oof_nusvc))
    print('nusvc2', roc_auc_score(train_df['target'], oof_nusvc2))
    print('qda', roc_auc_score(train_df['target'], oof_qda))
    print('svc', roc_auc_score(train_df['target'], oof_svc))
    print('knn', roc_auc_score(train_df['target'], oof_knn))
    print('knn', roc_auc_score(train_df['target'], oof_lr))
    
    return oof_train, oof_test
oof_train_1, oof_test_1 = get_oofs(42)
oof_train_4, oof_test_4 = get_oofs_2(42)
x_train_second_layer = oof_train_1  + oof_train_4
x_test_second_layer = oof_test_1  + oof_test_4
print('Ensemble', roc_auc_score(train_df['target'], x_train_second_layer.mean(1)))<jupyter_output>Ensemble 0.9738703857899418<jupyter_text>아래 SEED 수정 된 거로 바꿔야 함.<jupyter_code>submit = pd.read_csv('../input/sample_submission.csv')
submit["target"] = x_test_second_layer.mean(1)
submit.to_csv("submission.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13017789480444417, 'fraction_numerical': 0.014193051184894997, 'mean_word_length': 3.0946621088614252, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '330424', 'license': 'MIT', 'n_tokens_mistral': 6133, 'n_tokens_neox': 5881, 'n_words': 881, 'path': 'Instant Gratification/code/Model/Best.ipynb', 'repo_name': 'choco9966/kaggle', 'size': '15571'}