<jupyter_start><jupyter_text>3. FE_XGBClassifier_KFoldCVKaggle score:pip install jupyter notebook tqdm pillow h5py seaborn scikit-learn scikit-image xgboostAbstract:- date 7, 8, 9少feature的数据  Run name<jupyter_code>import time

project_name = 'TalkingdataAFD2018'
step_name = 'FE_XGBClassifier_KFoldCV'
time_str = time.strftime("%Y%m%d_%H%M%S", time.localtime())
run_name = '%s_%s_%s' % (project_name, step_name, time_str)
print('run_name: %s' % run_name)
t0 = time.time()<jupyter_output><empty_output><jupyter_text>Important params<jupyter_code>feature_run_name = 'TalkingdataAFD2018_FeatureExtraction_20180501_185800'
date = 100
print('date: ', date)


is_debug = False
print('is_debug: %s' % is_debug)

# epoch = 3
# batch_size = 2000 * 10000
# skip_data_len = (epoch - 1) * batch_size
# data_len = batch_size
# print('Echo: %s, Data rows: [%s, %s]' % (epoch, skip_data_len, skip_data_len + data_len))

# epoch = 2
# batch_size = 4000 * 10000
# skip_data_len = 59633310 - batch_size
# data_len = batch_size
# print('batch_size: %s' % batch_size)
# print('epoch: %s, data rows: [%s, %s]' % (epoch, skip_data_len, skip_data_len + data_len))

# run_name = '%s_date%s%s' % (run_name, date, epoch)
run_name = '%s_date%s' % (run_name, date)

print(run_name)

if is_debug:
    test_n_rows = 1 * 10000
else:
    test_n_rows = None
#     test_n_rows = 18790469
day_rows = {
    0: {
        'n_skiprows': 1,
        'n_rows': 1 * 1000
    },
    1: {
        'n_skiprows': 1 * 1000,
        'n_rows': 2 * 1000
    },
    6: {
        'n_skiprows': 1,
        'n_rows': 2 * 1000 # 9308568
    },
    7: {
        'n_skiprows': 1 + 9308568,
        'n_rows': 2 * 1000 # 59633310
    },
    8: {
        'n_skiprows': 1 + 9308568 + 59633310,
        'n_rows': 2 * 1000 # 62945075
    },
    9: {
        'n_skiprows': 1 + 9308568 + 59633310 + 62945075,
        'n_rows': 2 * 1000 # 53016937
    }
}
# n_skiprows = day_rows[date]['n_skiprows']
# n_rows = day_rows[date]['n_rows']<jupyter_output><empty_output><jupyter_text>Import PKGs<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
from IPython.display import display

import os
import sys
import gc
import time
import random
import zipfile
import h5py
import pickle
import math
from PIL import Image
import shutil

from tqdm import tqdm
import multiprocessing
from multiprocessing import cpu_count

from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import accuracy_score

random_num = np.random.randint(10000)
cpu_amount = cpu_count()

print('cpu_amount: %s' % (cpu_amount - 2))
print('random_num: %s' % random_num)<jupyter_output><empty_output><jupyter_text>Project folders<jupyter_code>cwd = os.getcwd()

input_folder = os.path.join(cwd, 'input')
output_folder = os.path.join(cwd, 'output')
model_folder = os.path.join(cwd, 'model')
feature_folder = os.path.join(cwd, 'feature')
log_folder = os.path.join(cwd, 'log')
print('input_folder: \t\t\t%s' % input_folder)
print('output_folder: \t\t\t%s' % output_folder)
print('model_folder: \t\t\t%s' % model_folder)
print('feature_folder: \t\t%s' % feature_folder)
print('log_folder: \t\t\t%s' % log_folder)

train_csv_file = os.path.join(input_folder, 'train.csv')
train_sample_csv_file = os.path.join(input_folder, 'train_sample.csv')
test_csv_file = os.path.join(input_folder, 'test.csv')
sample_submission_csv_file = os.path.join(input_folder, 'sample_submission.csv')

print('\ntrain_csv_file: \t\t%s' % train_csv_file)
print('train_sample_csv_file: \t\t%s' % train_sample_csv_file)
print('test_csv_file: \t\t\t%s' % test_csv_file)
print('sample_submission_csv_file: \t%s' % sample_submission_csv_file)<jupyter_output><empty_output><jupyter_text>Load feature<jupyter_code>sample_submission_csv = pd.read_csv(sample_submission_csv_file)
print('sample_submission_csv.shape: \t', sample_submission_csv.shape)
display(sample_submission_csv.head(2))

print('train_csv: %.2f Mb' % (sys.getsizeof(sample_submission_csv)/1024./1024.))
def save_feature(x_data, y_data, file_name):
    print(y_data[:5])
    if os.path.exists(file_name):
        os.remove(file_name)
        print('File removed: %s' % file_name)
    with h5py.File(file_name) as h:
        h.create_dataset('x_data', data=x_data)
        h.create_dataset('y_data', data=y_data)
    print('File saved:   %s' % file_name)

def load_feature(file_name):
    with h5py.File(file_name, 'r') as h:
        x_data = np.array(h['x_data'])
        y_data = np.array(h['y_data'])
    print('File loaded:  %s' % file_name)
    print(y_data[:5])
    
    return x_data, y_data


def save_test_feature(x_test, click_ids, file_name):
    print(click_ids[:5])
    if os.path.exists(file_name):
        os.remove(file_name)
        print('File removed: %s' % file_name)
    with h5py.File(file_name) as h:
        h.create_dataset('x_test', data=x_test)
        h.create_dataset('click_ids', data=click_ids)
    print('File saved:   %s' % file_name)

def load_test_feature(file_name):
    with h5py.File(file_name, 'r') as h:
        x_test = np.array(h['x_test'])
        click_ids = np.array(h['click_ids'])
    print('File loaded:  %s' % file_name)
    print(click_ids[:5])
    
    return x_test, click_ids
def save_feature_map(feature_map, file_name):
    print(feature_map[:5])
    feature_map_encode = []
    for item in feature_map:
        feature_name_encode = item[1].encode('UTF-8')
        feature_map_encode.append((item[0], feature_name_encode))
    if os.path.exists(file_name):
        os.remove(file_name)
        print('File removed: \t%s' % file_name)
    with h5py.File(file_name) as h:
        h.create_dataset('feature_map', data=feature_map_encode)
    print('File saved: \t%s' % file_name)

def load_feature_map(file_name):
    with h5py.File(file_name, 'r') as h:
        feature_map_encode = np.array(h['feature_map'])
    print('File loaded: \t%s' % file_name)
    feature_map = []
    for item in feature_map_encode:
        feature_name = item[1].decode('UTF-8')
        feature_map.append((int(item[0]), feature_name))
    print(feature_map[:5])
    
    return feature_map
def describe(data):
    if isinstance(data, list):
        print(len(data), '\t\t%.2f Mb' % (sys.getsizeof(data)/1024./1024.))
    else:
        print(data.shape, '\t%.2f Mb' % (sys.getsizeof(data)/1024./1024.))

test_np = np.ones((5000, 10))
describe(test_np)
describe(list(range(5000)))
%%time

feature_files = []
x_train = []
y_train = []
if date == 100:
    for key in [7, 8, 9]:
        y_proba_file = os.path.join(feature_folder, 'feature_%s_date%s.p' % (feature_run_name, key))
        feature_files.append(y_proba_file)
        x_train_date, y_train_date = load_feature(y_proba_file)
        x_train.append(x_train_date)
        y_train.append(y_train_date)
    x_train = np.concatenate(x_train, axis=0)
    y_train = np.concatenate(y_train, axis=0)
else:
    y_proba_file = os.path.join(feature_folder, 'feature_%s_date%s.p' % (feature_run_name, date))
    feature_files.append(y_proba_file)
    x_train, y_train = load_feature(y_proba_file)

# Use date 6 as validation dataset
y_proba_file = os.path.join(feature_folder, 'feature_%s_date%s.p' % (feature_run_name, 6))
feature_files.append(y_proba_file)
x_val, y_val = load_feature(y_proba_file)



print('*' * 80)
describe(x_train)
describe(y_train)
describe(x_val)
describe(y_val)

%%time

# from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle

# x_train, x_val, y_train, y_val = train_test_split(x_data[skip_data_len: data_len], y_data[skip_data_len: data_len], test_size=0.1, random_state=random_num, shuffle=True)
# x_train, x_val, y_train, y_val = train_test_split(x_data, y_data, test_size=0.2, random_state=random_num, shuffle=False)

# x_train = x_train[skip_data_len: skip_data_len + data_len]
# y_train = y_train[skip_data_len: skip_data_len + data_len]

x_train, y_train = shuffle(x_train, y_train, random_state=random_num)
describe(x_train)
describe(y_train)
describe(x_val)
describe(y_val)
gc.collect()<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>%%time
import warnings
warnings.filterwarnings('ignore')

from xgboost import XGBClassifier, Booster
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score
from sklearn.cross_validation import KFold
def remove_file(file_path, is_print=False):
    if os.path.exists(file_path):
        os.remove(file_path)
        if is_print:
            print('File removed: %s' % file_path)
    else:
        if is_print:
            print('File do not exists: %s' % file_path)
def get_temp_model_file(kfold_indx=None):
    if kfold_indx is None:
        return os.path.join(model_folder, 'temp_%s.xgb_model' % run_name)
    return os.path.join(model_folder, 'temp_%s_p%s.xgb_model' % (run_name, kfold_indx))

def get_model_file(kfold_indx=None):
    if kfold_indx is None:
        return os.path.join(model_folder, '%s.xgb_model' % run_name)
    return os.path.join(model_folder, '%s_p%s.xgb_model' % (run_name, kfold_indx))

def get_xgb_model_from_file(file_path, is_print=True):
    model = XGBClassifier(
        max_depth=3, 
        learning_rate=0.1, 
        n_estimators=1000, 
        silent=True, 
        objective='gpu:binary:logistic', 
        booster='gbtree', 
        n_jobs=cpu_amount, 
        nthread=None, 
        gamma=0, 
        min_child_weight=1, 
        max_delta_step=0, 
        subsample=0.8, 
        colsample_bytree=1, 
        colsample_bylevel=1, 
        reg_alpha=1, 
        reg_lambda=2, 
        scale_pos_weight=97, 
        base_score=0.5, 
        random_state=random_num, 
        seed=None, 
        missing=None,
        # booster params
        num_boost_round=60,
        early_stopping_rounds=30,
        tree_method='gpu_hist',
        predictor='gpu_predictor',
        eval_metric=['auc'],
        verbose_eval=False,
#         n_gpus=8,
    )
    booster = Booster()
    booster.load_model(file_path)
    model._Booster = booster
    if is_print:
        print('XGBoost model loaded: %s' % file_path)
    return model

def partial_predict_proba(model, x_data, n_folds=100):
    kf = KFold(len(x_data), n_folds=n_folds)
    y_data_probas = []
    print('|', end='')
    for train_index, val_index in kf:
        print('*', end='')
        y_data_proba_fold = model.predict_proba(x_data[val_index])
        y_data_probas.append(y_data_proba_fold)
    y_data_probas = np.concatenate(y_data_probas, axis=0)
    print('|')
    return y_data_probas[:, 1]

def partial_validation(model, x_data, y_data):
    y_data_proba = partial_predict_proba(model, x_data)
    y_data_pred = (y_data_proba>=0.5).astype(int)
    acc = accuracy_score(y_data, y_data_pred)
    roc = roc_auc_score(y_data, y_data_proba)
    return acc, roc

def partial_fit(x_train, y_train, x_val, y_val, temp_xgb_model_file, batch_size=10*10000):
    clf = XGBClassifier(
        max_depth=3, 
        learning_rate=0.1, 
        n_estimators=1000, 
        silent=True, 
        objective='gpu:binary:logistic', 
        booster='gbtree', 
        n_jobs=cpu_amount, 
        nthread=None, 
        gamma=0, 
        min_child_weight=1, 
        max_delta_step=0, 
        subsample=0.8, 
        colsample_bytree=1, 
        colsample_bylevel=1, 
        reg_alpha=1, 
        reg_lambda=2, 
        scale_pos_weight=97, 
        base_score=0.5, 
        random_state=random_num, 
        seed=None, 
        missing=None,
        # booster params
        num_boost_round=60,
        early_stopping_rounds=30,
        tree_method='gpu_hist',
        predictor='gpu_predictor',
        eval_metric=['auc'],
        verbose_eval=False,
#         n_gpus=8,
    )

    len_train = len(x_train)
    n_folds = len_train // batch_size
    print('continue_XGBC: len(x_train)=%s, n_folders=%s' % (len_train, n_folds))
    kf = KFold(len_train, n_folds=n_folds)
    epoch = 2
    for j in range(epoch):
        x_train, y_train = shuffle(x_train, y_train, random_state=None)
        for i, indxes in enumerate(kf):
            print('%s/%s' % (i+1, n_folds), end='')
            kf_start = time.time()
            train_index = indxes[0]
            val_index = indxes[1]
            x_train_patial = x_train[val_index]
            y_train_patial = y_train[val_index]
            # describe(x_train_patial)
            # describe(y_train_patial)
            # describe(x_val_patial)
            # describe(y_val_patial)

            temp_model_file = None
            if os.path.exists(temp_xgb_model_file):
                temp_model_file = temp_xgb_model_file
                print('+', end='')
    #             print('xgb_model loaded: %s' % temp_model_file)
            else:
                print('-', end='')
    #             print('File do not exists: %s' % temp_model_file)
            clf.fit(
                x_train_patial, 
                y_train_patial,
            #     sample_weight=None, 
                eval_set=[(x_train_patial, y_train_patial)], 
                eval_metric=['auc'], 
                early_stopping_rounds=20, 
                verbose=False, 
                xgb_model=temp_model_file
            )
            clf.get_booster().save_model(temp_xgb_model_file)
            del train_index
            del val_index
            del x_train_patial
            del y_train_patial
            # print('saved xgb_model')
            kf_time_cost = time.time() - kf_start
            print(': %.2fs...' % kf_time_cost, end='')
            gc.collect()
            time.sleep(2)
            gc.collect()
        print('')

        acc_train, roc_train = partial_validation(clf, x_train, y_train)
        acc_val, roc_val = partial_validation(clf, x_val, y_val)
        print((acc_train, roc_train, acc_val, roc_val))
    del clf
    del x_train
    del y_train
    del x_val
    del y_val
    return acc_train, roc_train, acc_val, roc_val

def fit(x_train, y_train, x_val, y_val, n_folds=4, batch_size=10*10000):
    len_train = len(x_train)
    n_folds = n_folds
    print('kfold_XGBC: len(x_train)=%s, n_folders=%s' % (len_train, n_folds))
    kf = KFold(len_train, n_folds=n_folds)
    
    results = []
    for i, indxes in enumerate(kf):
        print('KFold: %s/%s' % (i+1, n_folds))
        temp_model_file = get_temp_model_file(i)
        remove_file(temp_model_file, True)
        
        train_index = indxes[0]
        val_index = indxes[1]
        kf_start = time.time()
        x_train_patial = x_train[train_index]
        y_train_patial = y_train[train_index]
        x_val_patial = x_train[val_index]
        y_val_patial = y_train[val_index]
        # describe(x_train_patial)
        # describe(y_train_patial)
        # describe(x_val_patial)
        # describe(y_val_patial)
        acc_train, roc_train, acc_val, roc_val = partial_fit(x_train_patial, y_train_patial, x_val_patial, y_val_patial, temp_model_file, batch_size)
        model = get_xgb_model_from_file(temp_model_file)
        kfold_acc_val1, kfold_roc_val1 = partial_validation(model, x_val, y_val)
        model_file = get_model_file(i)
        model.get_booster().save_model(model_file)
        time_cost = time.time() - kf_start
        kfold_result = {
            'model_file': model_file,
            'acc_train': acc_train,
            'roc_train': roc_train,
            'acc_val': acc_val,
            'roc_val': roc_val,
            'time_cost': time_cost
        }
        results.append(kfold_result)
        remove_file(temp_model_file)
        del train_index
        del val_index
        del x_train_patial
        del y_train_patial
        del x_val_patial
        del y_val_patial
        gc.collect()
        time.sleep(2)
        gc.collect()
        print('model_file: %s, acc_train: %.4f, roc_train: %.4f, acc_val: %.4f, roc_val: %.4f, kfold_acc_val: %.4f, kfold_roc_val: %.4f, time_cost: %.2f s' \
                % (model_file, acc_train, roc_train, acc_val, roc_val, kfold_acc_val, kfold_roc_val, time_cost))
    return results

def get_model_files():
    file_names = os.listdir(model_folder)
    model_files = []
    for file_name in file_names:
        if file_name.endswith('.xgb_model') and file_name.startswith('%s' % run_name):
            file_path = os.path.join(model_folder, file_name)
            model_files.append(file_path)

    return model_files

def predict_proba(x_test):
    y_data_probas = []
    model_files = get_model_files()
    if len(model_files) == 0:
        raise Exception('Do not any find xgboost model file!')
    for model_file in model_files:
        model = get_xgb_model_from_file(model_file)
        y_data_proba = partial_predict_proba(model, x_test)
        y_data_proba = y_data_proba[:, np.newaxis]
        print(y_data_proba.shape)
        y_data_probas.append(y_data_proba)
    y_data_probas = np.concatenate(y_data_probas, axis=-1)
    print(y_data_probas.shape)
    y_data_probas_mean = np.mean(y_data_probas, axis=-1)
    print(y_data_probas_mean.shape)
    return y_data_probas_mean

def ensemble_validation(x_data, y_data):
    y_data_proba = predict_proba(x_data)
    y_data_pred = (y_data_proba>=0.5).astype(int)
    acc = accuracy_score(y_data, y_data_pred)
    roc = roc_auc_score(y_data, y_data_proba)
    return acc, roc

def validation(y_data, y_data_proba):
    y_data_pred = (y_data_proba>=0.5).astype(int)
    acc = accuracy_score(y_data, y_data_pred)
    roc = roc_auc_score(y_data, y_data_proba)
    return acc, roc

data_rows = 200 #len(x_train)
batch_size = 100 * 10000

results = fit(x_train[:data_rows], y_train[:data_rows], x_val, y_val, 2, batch_size)
y_val_proba = predict_proba(x_val)
acc_val, roc_val = validation(y_val, y_val_proba)
describe(y_val_proba)

print('acc_val: %.4f, roc_val: %.4f' % (acc_val, roc_val))<jupyter_output><empty_output><jupyter_text>Feature importance<jupyter_code>xgboost_model = get_xgb_model_from_file(get_model_files()[0])
from xgboost import plot_importance

fig, ax = plt.subplots(figsize=(10,int(x_train.shape[1]/2)))
plot_importance(xgboost_model, height=0.5, ax=ax, max_num_features=300)
plt.show()
feature_map_file_name = os.path.join(feature_folder, 'feature_map_TalkingdataAFD2018_FeatureExtraction_test_20180501_143516.p')

feature_map = load_feature_map(feature_map_file_name)
print(len(feature_map))
print(feature_map[:5])

feature_dict = {}
for item in feature_map:
    feature_dict[item[0]] = item[1]
print(list(feature_dict.keys())[:5])
print(list(feature_dict.values())[:5])
# print(dir(grid_search.best_estimator_.get_booster()))
importance_score = xgboost_model.get_booster().get_fscore()
sorted_score = []
for key in importance_score:
    indx = int(key[1:])
    sorted_score.append((importance_score[key], key, indx, feature_dict[indx]))
dtype = [('importance_score', int), ('key', 'S50'), ('indx', int), ('name', 'S50')]
importance_table = np.array(sorted_score, dtype=dtype)
display(importance_table[:2])
importance_table = np.sort(importance_table, axis=0, order=['importance_score'])
importance_table = importance_table[::-1]
display(importance_table)
del x_train
del y_train
gc.collect()<jupyter_output><empty_output><jupyter_text>Predict<jupyter_code>run_name_acc = run_name + '_' + str(int(roc_val*10000)).zfill(4)
print(run_name_acc)
%%time
y_proba_file = os.path.join(feature_folder, 'feature_%s_test.p' % feature_run_name)
feature_files.append(y_proba_file)
x_test, click_ids = load_test_feature(y_proba_file)

describe(x_test)
describe(click_ids)
print(feature_files)
y_test_proba = predict_proba(x_test)

print(y_test_proba.shape)
print(y_test_proba[:20])
def save_proba(y_val_proba, y_val, y_test_proba, click_ids, file_name):
    print(click_ids[:5])
    if os.path.exists(file_name):
        os.remove(file_name)
        print('File removed: %s' % file_name)
    with h5py.File(file_name) as h:
#         h.create_dataset('y_train_proba', data=y_train_proba)
#         h.create_dataset('y_train', data=y_train)
        h.create_dataset('y_val_proba', data=y_val_proba)
        h.create_dataset('y_val', data=y_val)
        h.create_dataset('y_test_proba', data=y_test_proba)
        h.create_dataset('click_ids', data=click_ids)
    print('File saved:   %s' % file_name)

def load_proba(file_name):
    with h5py.File(file_name, 'r') as h:
#         y_train_proba = np.array(h['y_train_proba'])
#         y_train = np.array(h['y_train'])
        y_val_proba = np.array(h['y_val_proba'])
        y_val = np.array(h['y_val'])
        y_test_proba = np.array(h['y_test_proba'])
        click_ids = np.array(h['click_ids'])
    print('File loaded:  %s' % file_name)
    print(click_ids[:5])
    
    return y_val_proba, y_val, y_test_proba, click_ids


y_proba_file = os.path.join(model_folder, 'proba_%s.p' % run_name_acc)
print(y_proba_file)
save_proba(
#     y_train_proba, 
#     y_train, 
    y_val_proba, 
    y_val, 
    y_test_proba, 
    np.array(sample_submission_csv['click_id']), 
    y_proba_file
)
y_val_proba_true, y_val, y_test_proba_true, click_ids = load_proba(y_proba_file)

# print(y_train_proba_true.shape)
# print(y_train.shape)
print(y_val_proba_true.shape)
print(y_val.shape)
print(y_test_proba_true.shape)
print(len(click_ids))
# %%time
submission_csv_file = os.path.join(output_folder, 'pred_%s.csv' % run_name_acc)
print(submission_csv_file)
submission_csv = pd.DataFrame({ 'click_id': click_ids , 'is_attributed': y_test_proba_true })
submission_csv.to_csv(submission_csv_file, index = False)
display(submission_csv.head())
print('Time cost: %.2f s' % (time.time() - t0))

print('random_num: ', random_num)
print('date: ', date)
print(run_name_acc)
print('Done!')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12188813449725186, 'fraction_numerical': 0.020507135929056394, 'mean_word_length': 3.617615696310514, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '612092', 'license': 'MIT', 'n_tokens_mistral': 8649, 'n_tokens_neox': 8287, 'n_words': 1357, 'path': 'talkingdata-adtracking-fraud-detection/3. FE_XGBClassifier_KFoldCV.ipynb', 'repo_name': 'StudyExchange/Kaggle', 'size': '21651'}