<jupyter_start><jupyter_text>사전작업  라이브러리 로드<jupyter_code>import numpy as np
import pandas as pd
import warnings
import gc
from tqdm import tqdm_notebook
import lightgbm as lgb
from scipy.sparse import vstack, csr_matrix, save_npz, load_npz
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.model_selection import StratifiedKFold
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.compose import ColumnTransformer

warnings.filterwarnings("ignore")
gc.enable()
pd.set_option('max_rows', 150)
pd.set_option('max_colwidth', 500)
pd.set_option('max_columns', 500)<jupyter_output><empty_output><jupyter_text>데이터 로드<jupyter_code>dtypes = {
        'MachineIdentifier':                                    'object',
        'ProductName':                                          'category',
        'EngineVersion':                                        'category',
        'AppVersion':                                           'category',
        'AvSigVersion':                                         'category',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'category',
        'Processor':                                            'category',
        'OsVer':                                                'category',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'category',
        'OsBuildLab':                                           'category',
        'SkuEdition':                                           'category',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'category',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'category',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'category',
        'Census_DeviceFamily':                                  'category',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'category',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'category',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'category',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'category',
        'Census_InternalBatteryType':                           'category',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'category',
        'Census_OSArchitecture':                                'category',
        'Census_OSBranch':                                      'category',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'category',
        'Census_OSSkuName':                                     'category',
        'Census_OSInstallTypeName':                             'category',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'category',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'category',
        'Census_ActivationChannel':                             'category',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'category',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }
%%time
train = pd.read_csv('./data/train.csv', dtype=dtypes)
test = pd.read_csv('./data/test.csv', dtype=dtypes)<jupyter_output>Wall time: 3min 41s<jupyter_text>train, test 합치기<jupyter_code>test['HasDetections'] = np.nan
data = train.append(test)
data.reset_index(drop=True, inplace=True)
data = data.reset_index().drop(['MachineIdentifier'], axis=1).rename(columns={'index':'MachineIdentifier'})
del train, test
gc.collect()<jupyter_output><empty_output><jupyter_text>Get Date - AvSigVersion<jupyter_code>datedict = np.load('./data/AvSigVersionTimestamps.npy')
datedict = datedict[()]
data['Date'] = data['AvSigVersion'].map(datedict)
data['Date_YMD'] = pd.to_datetime(data['Date'].astype(str).str.slice(0, 10))
del datedict<jupyter_output><empty_output><jupyter_text>2018년 7월 25일 ~ 9월 24일  Data Load<jupyter_code>train = data[~data.HasDetections.isna()]
test = data[data.HasDetections.isna()]
lb_692 = pd.read_csv('./data/submission_lgb.csv')
lb_692['Date_YMD'] = test.Date_YMD.values
lb_701 = pd.read_csv('./data/YH_v3_new_jebal.csv')
lb_701['Date_YMD'] = test.Date_YMD.values
lb_702 = pd.read_csv('./data/time_test.csv')
lb_702['Date_YMD'] = test.Date_YMD.values
lb_706 = pd.read_csv('./data/time_test_3.csv')
lb_706['Date_YMD'] = test.Date_YMD.values
catboost_timeshift = pd.read_csv('./data/catboost_timeshift.csv')
catboost_timeshift['Date_YMD'] = test.Date_YMD.values
catboost_timeshift_post = pd.read_csv('./data/catboost_timeshift_postprocessing.csv')
catboost_timeshift_post['Date_YMD'] = test.Date_YMD.values<jupyter_output><empty_output><jupyter_text>make meta<jupyter_code>time_idx = data.groupby(['Date_YMD']).size().index
temp = pd.DataFrame(time_idx)
condition = (temp.Date_YMD >= '2018-07-25')&(temp.Date_YMD <= '2018-11-24')<jupyter_output><empty_output><jupyter_text>train, test<jupyter_code>tr_temp = train.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
te_temp = test.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
tr_temp = temp.merge(tr_temp, on='Date_YMD', how='left')
te_temp = temp.merge(te_temp, on='Date_YMD', how='left')
tr_temp = tr_temp[condition]
te_temp = te_temp[condition]<jupyter_output><empty_output><jupyter_text>lb_692<jupyter_code>lb_temp_692 = lb_692.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
lb_temp_692 = temp.merge(lb_temp_692, on='Date_YMD', how='left')
lb_temp_692 = lb_temp_692[condition]<jupyter_output><empty_output><jupyter_text>lb_701<jupyter_code>lb_temp_701 = lb_701.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
lb_temp_701 = temp.merge(lb_temp_701, on='Date_YMD', how='left')
lb_temp_701 = lb_temp_701[condition]<jupyter_output><empty_output><jupyter_text>lb_702<jupyter_code>lb_temp_702 = lb_702.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
lb_temp_702 = temp.merge(lb_temp_702, on='Date_YMD', how='left')
lb_temp_702 = lb_temp_702[condition]<jupyter_output><empty_output><jupyter_text>lb_706<jupyter_code>lb_temp_706 = lb_706.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
lb_temp_706 = temp.merge(lb_temp_706, on='Date_YMD', how='left')
lb_temp_706 = lb_temp_706[condition]<jupyter_output><empty_output><jupyter_text>catboost_timeshift<jupyter_code>catboost_timeshift_temp = catboost_timeshift.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
catboost_timeshift_temp = temp.merge(catboost_timeshift_temp, on='Date_YMD', how='left')
catboost_timeshift_temp = catboost_timeshift_temp[condition]<jupyter_output><empty_output><jupyter_text>catboost_timeshift_post<jupyter_code>catboost_timeshift_post_temp = catboost_timeshift_post.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
catboost_timeshift_post_temp = temp.merge(catboost_timeshift_post_temp, on='Date_YMD', how='left')
catboost_timeshift_post_temp = catboost_timeshift_post_temp[condition]<jupyter_output><empty_output><jupyter_text>hypothesis<jupyter_code>hypothesis_temp = train.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
hypothesis_temp.Date_YMD = hypothesis_temp.Date_YMD + pd.offsets.MonthOffset(2)
condition = (hypothesis_temp.Date_YMD >= '2018-07-25')&(hypothesis_temp.Date_YMD <= '2018-09-24')
hypothesis_temp = hypothesis_temp[condition]
hypothesis_temp = temp.merge(hypothesis_temp, on='Date_YMD', how='left')<jupyter_output><empty_output><jupyter_text>check_hypothesis<jupyter_code># test = data[data.HasDetections.isna()]
# test = test[(test.Date_YMD >= '2018-09-26') & (test.Date_YMD <= '2018-11-25')]
# test.HasDetections = lgb_test_result / 3
# check_temp = test.groupby(['Date_YMD']).HasDetections.agg(['size', 'mean']).reset_index()
# check_temp2 = check.groupby(['Date_YMD']).HasDetections.agg(['size', 'mean']).reset_index()
# condition = (check_temp2.Date_YMD >= '2018-09-26')&(check_temp2.Date_YMD <= '2018-11-25')
# check_temp2 = check_temp[condition]<jupyter_output><empty_output><jupyter_text>temp<jupyter_code># temp_temp = temp.groupby('Date_YMD').HasDetections.agg(['size', 'mean']).reset_index()
# temp_temp = temp.merge(temp_temp, on='Date_YMD', how='left')
# vmffhtemp_temp = temp_temp[condition]<jupyter_output><empty_output><jupyter_text>plot<jupyter_code>plt.figure(figsize=(18, 8))
plt.plot(tr_temp.Date_YMD, tr_temp['mean'], 'o', color='b', alpha=1, label='train')
# plt.plot(lb_temp_692.Date_YMD, lb_temp_692['mean'], 'o', color='r', alpha=1, label='692')
# plt.plot(lb_temp_701.Date_YMD, lb_temp_701['mean'], 'o', color='g', alpha=1, label='701')
# plt.plot(lb_temp_702.Date_YMD, lb_temp_702['mean'], 'o', color='y', alpha=1, label='702')
plt.plot(lb_temp_706.Date_YMD, lb_temp_706['mean'], 'o', color='k', alpha=1, label='706')
# plt.plot(check_temp2.Date_YMD, check_temp2['mean'], 'o', color='c', alpha=1, label='700')
# plt.plot(check_temp.Date_YMD, check_temp['mean'], 'o', color='b', alpha=0.2, label='694')
# plt.plot(temp_temp.Date_YMD, temp_temp['mean'], 'o', color='r', alpha=0.2, label='694')
plt.plot(hypothesis_temp.Date_YMD, hypothesis_temp['mean'], '+', color='c', alpha=1, label='hypothesis')
plt.plot(catboost_timeshift_temp.Date_YMD, catboost_timeshift_temp['mean'], '+', color='g', alpha=1, label='catboost')
plt.plot(catboost_timeshift_post_temp.Date_YMD, catboost_timeshift_post_temp['mean'], '+', color='y', alpha=1, label='catboost_prepos')
plt.legend(prop={'size': 16})
plt.figure(figsize=(18, 8))
plt.plot(tr_temp.Date_YMD, tr_temp['size'], 'o', color='b', alpha=1, label='train')
plt.plot(te_temp.Date_YMD, te_temp['size'], 'o', color='r', alpha=1, label='train')
plt.plot(hypothesis_temp.Date_YMD, hypothesis_temp['size'], 'o', color='c', alpha=1, label='train')
plt.legend(prop={'size': 16})<jupyter_output><empty_output><jupyter_text>2018-10-15일부터 의도적으로 낮춰보기<jupyter_code># 저점 = 0.38
def get_has_detections(data):
    try:
        if data.Date_YMD == '2018-10-25':
            return data.HasDetections * 0.8932461873638343
        elif data.Date_YMD == '2018-10-24':
            return data.HasDetections *  0.946000733633222
        elif data.Date_YMD == '2018-10-23':
            return data.HasDetections *  0.9557970983024938
        elif data.Date_YMD == '2018-10-22':
            return data.HasDetections * 0.9655259079294601
        elif data.Date_YMD == '2018-10-21':
            return data.HasDetections * 0.9681535196547988
        elif data.Date_YMD == '2018-10-20':
            return data.HasDetections * 0.9655259079294601
        else:
            return data.HasDetections
    except:
        print(data)
t1 = lb_706[lb_706.Date_YMD == '2018-10-25']
t2 = lb_706[lb_706.Date_YMD == '2018-10-24']
t3 = lb_706[lb_706.Date_YMD == '2018-10-23']
t4 = lb_706[lb_706.Date_YMD == '2018-10-22']
t5 = lb_706[lb_706.Date_YMD == '2018-10-21']
t6 = lb_706[lb_706.Date_YMD == '2018-10-20']
t7 = lb_706[~((lb_706.Date_YMD == '2018-10-25') | (lb_706.Date_YMD == '2018-10-24') | (lb_706.Date_YMD == '2018-10-23') | (lb_706.Date_YMD == '2018-10-22')| (lb_706.Date_YMD == '2018-10-21') | (lb_706.Date_YMD == '2018-10-20'))]
t1.HasDetections = t1.HasDetections * 0.8732461873638343
t2.HasDetections = t2.HasDetections * 0.926000733633222
t3.HasDetections = t3.HasDetections * 0.9457970983024938
t4.HasDetections = t4.HasDetections * 0.9555259079294601
t5.HasDetections = t5.HasDetections * 0.9581535196547988
t6.HasDetections = t6.HasDetections * 0.9555259079294601
temp = t1.append(t2).append(t3).append(t4).append(t5).append(t6).append(t7).sort_values('MachineIdentifier').reset_index(drop=True)
temp[['MachineIdentifier', 'HasDetections']].to_csv('./data/time_test_6.csv', index=False)
temp = temp.groupby(['Date_YMD']).HasDetections.agg(['size','mean']).reset_index()
temp = temp[(temp.Date_YMD >= '2018-10-15') & (temp.Date_YMD <= '2018-10-25')]
plt.figure(figsize=(18,8))
plt.plot(temp.Date_YMD, temp['mean'], 'or')
plt.ylim(0.41, 0.55)
plt.figure(figsize=(18,8))
plt.plot(temp.Date_YMD, temp['mean'], 'or')
plt.ylim(0.41, 0.55)<jupyter_output><empty_output><jupyter_text>702에서 9월 18일까지를 Hypothesis Check으로 바꿔보기<jupyter_code>change = lb_702[(lb_702.Date_YMD >= '2018-07-25') & (lb_702.Date_YMD <= '2018-09-18')]
not_change = lb_702[~((lb_702.Date_YMD >= '2018-07-25') & (lb_702.Date_YMD <= '2018-09-18'))]
test = test[['MachineIdentifier', 'HasDetections', 'Date_YMD']]
test = test[(test.Date_YMD >= '2018-07-25') & (test.Date_YMD <= '2018-09-18')]
change.HasDetections = test.HasDetections.values
check = change.append(not_change).sort_values('MachineIdentifier')
check[['MachineIdentifier', 'HasDetections']].to_csv('./data/time_test_2.csv', index=False)<jupyter_output><empty_output><jupyter_text>706에서 9월 26일부터 11월 25일까지 Hypothesis Check으로 바꿔보기<jupyter_code>change = lb_706[(lb_706.Date_YMD >= '2018-09-26') & (lb_706.Date_YMD <= '2018-11-25')]
not_change = lb_706[~((lb_706.Date_YMD >= '2018-09-26') & (lb_706.Date_YMD <= '2018-11-25'))]
test = test[['MachineIdentifier', 'HasDetections', 'Date_YMD']]
test = test[(test.Date_YMD >= '2018-09-26') & (test.Date_YMD <= '2018-11-25')]
change.HasDetections = test.HasDetections.values
check = change.append(not_change).sort_values('MachineIdentifier')
check[['MachineIdentifier', 'HasDetections']].to_csv('./data/time_test_4.csv', index=False)<jupyter_output><empty_output><jupyter_text>Crazy Method - 2달 미뤄서 훈련시켜보기<jupyter_code>temp = data[(data.Date_YMD >= '2018-05-25') & (data.Date_YMD <= '2018-09-24')]
del temp['Date']
gc.collect()
train = temp[~temp.HasDetections.isna()]
test = temp[temp.HasDetections.isna()]

del temp
gc.collect()
train = train[train.Date_YMD <= '2018-07-24']
test = test[test.Date_YMD >= '2018-07-25']

gc.collect()
# 2달 미루기
train.Date_YMD = train.Date_YMD + pd.offsets.MonthOffset(2)
train.reset_index(drop=True, inplace=True)
test.reset_index(drop=True, inplace=True)
train.shape
test.shape
del train['AvSigVersion'], test['AvSigVersion']
gc.collect()
print('Transform all features to category.\n')
for usecol in tqdm_notebook([col for col in train.columns if col not in ['HasDetections', 'MachineIdentifier']]):

    train[usecol] = train[usecol].astype('str')
    test[usecol] = test[usecol].astype('str')
    
    #Fit LabelEncoder
    le = LabelEncoder().fit(
            np.unique(train[usecol].unique().tolist()+
                      test[usecol].unique().tolist()))

    #At the end 0 will be used for dropped values
    train[usecol] = le.transform(train[usecol])+1
    test[usecol]  = le.transform(test[usecol])+1

    agg_tr = (train
              .groupby([usecol])
              .aggregate({'MachineIdentifier':'count'})
              .reset_index()
              .rename({'MachineIdentifier':'Train'}, axis=1))
    
    agg_te = (test
              .groupby([usecol])
              .aggregate({'MachineIdentifier':'count'})
              .reset_index()
              .rename({'MachineIdentifier':'Test'}, axis=1))

    agg = pd.merge(agg_tr, agg_te, on=usecol, how='outer').replace(np.nan, 0)

    #Select values with more than 1000 observations
    agg = agg[(agg['Train'] > 50)].reset_index(drop=True)
    agg['Total'] = agg['Train'] / 3 + agg['Test']
    #Drop unbalanced values
    agg = agg[(agg['Train'] / (agg['Total'] * 3) > 0.14) & (agg['Train'] / (agg['Total'] * 3) < 0.86)]
    agg[usecol+'Copy'] = agg[usecol]
        
    train[usecol] = (pd.merge(train[[usecol]], 
                              agg[[usecol, usecol+'Copy']], 
                              on=usecol, how='left')[usecol+'Copy']
                     .replace(np.nan, 0).astype('int').astype('category'))

    test[usecol]  = (pd.merge(test[[usecol]], 
                              agg[[usecol, usecol+'Copy']], 
                              on=usecol, how='left')[usecol+'Copy']
                     .replace(np.nan, 0).astype('int').astype('category'))

    del le, agg_tr, agg_te, agg, usecol
    gc.collect()
train.shape
y_train = np.array(train['HasDetections'])
train_ids = train.index
test_ids  = test.index
del train['HasDetections'], train['MachineIdentifier'], test['MachineIdentifier'], test['HasDetections']
gc.collect()
print("If you don't want use Sparse Matrix choose Kernel Version 2 to get simple solution.\n")

print('--------------------------------------------------------------------------------------------------------')
print('Transform Data to Sparse Matrix.')
print('Sparse Matrix can be used to fit a lot of models, eg. XGBoost, LightGBM, Random Forest, K-Means and etc.')
print('To concatenate Sparse Matrices by column use hstack()')
print('Read more about Sparse Matrix https://docs.scipy.org/doc/scipy/reference/sparse.html')
print('Good Luck!')
print('--------------------------------------------------------------------------------------------------------')

#Fit OneHotEncoder
ohe = OneHotEncoder(categories='auto', sparse=True, dtype='uint8').fit(train.append(test))
#Transform data using small groups to reduce memory usage
m = 100000
train = vstack([ohe.transform(train[i*m:(i+1)*m]) for i in range(train.shape[0] // m + 1)])
test  = vstack([ohe.transform(test[i*m:(i+1)*m])  for i in range(test.shape[0] // m +  1)])
train.shape
test.shape
save_npz('./data_temp/train.npz', train, compressed=True)
save_npz('./data_temp/test.npz',  test,  compressed=True)
del ohe, train, test
gc.collect()
train = load_npz('./data_temp/train.npz')
test = load_npz('./data_temp/test.npz')
test = csr_matrix(test, dtype='float32')
train.shape
test.shape
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
skf.get_n_splits(train_ids, y_train)

lgb_test_result  = np.zeros(test_ids.shape[0])
lgb_train_result = np.zeros(train_ids.shape[0])
counter = 0
print('\nLightGBM\n')

for train_index, test_index in skf.split(train_ids, y_train):
    
    print('Fold {}\n'.format(counter + 1))
    
    X_fit = vstack([train[train_index[i*m:(i+1)*m]] for i in range(train_index.shape[0] // m + 1)])
    X_val = vstack([train[test_index[i*m:(i+1)*m]]  for i in range(test_index.shape[0] //  m + 1)])
    X_fit, X_val = csr_matrix(X_fit, dtype='float32'), csr_matrix(X_val, dtype='float32')
    y_fit, y_val = y_train[train_index], y_train[test_index]
    
    gc.collect()

    lgb_model = lgb.LGBMClassifier(max_depth=-1,
                                   n_estimators=30000,
                                   learning_rate=0.05,
                                   num_leaves=2**5-1,
                                   objective='binary', 
                                   boosting_type='gbdt',
                                   # overfitting handling
                                   # max_bin=120,
                                   # lambda_l1=6,
                                   # lambda_l2=2,
                                   save_binary=True,
                                   feature_fraction=0.8,
                                   feature_fraction_seed=42,
                                   n_jobs=-1)
    
    print("fitting")
    lgb_model.fit(X_fit, y_fit, eval_metric='auc', 
                  eval_set=[(X_val, y_val)], 
                  verbose=100, early_stopping_rounds=100)
    
    del X_fit, X_val, y_fit, y_val, train_index, test_index
    gc.collect()
    
    print("predicting")
    lgb_test_result += lgb_model.predict_proba(test)[:,1]
    counter += 1
    
    gc.collect()<jupyter_output>LightGBM

Fold 1

fitting
Training until validation scores don't improve for 100 rounds.
[100]	valid_0's auc: 0.677371	valid_0's binary_logloss: 0.604701
[200]	valid_0's auc: 0.685048	valid_0's binary_logloss: 0.599969
[300]	valid_0's auc: 0.687145	valid_0's binary_logloss: 0.598676
[400]	valid_0's auc: 0.687966	valid_0's binary_logloss: 0.598151
[500]	valid_0's auc: 0.688526	valid_0's binary_logloss: 0.59779
[600]	valid_0's auc: 0.688837	valid_0's binary_logloss: 0.597577
[700]	valid_0's auc: 0.689194	valid_0's binary_logloss: 0.597344
[800]	valid_0's auc: 0.68942	valid_0's binary_logloss: 0.597182
[900]	valid_0's auc: 0.689547	valid_0's binary_logloss: 0.597075
[1000]	valid_0's auc: 0.689621	valid_0's binary_logloss: 0.596991
[1100]	valid_0's auc: 0.68967	valid_0's binary_logloss: 0.596924
[1200]	valid_0's auc: 0.689712	valid_0's binary_logloss: 0.596872
[1300]	valid_0's auc: 0.689751	valid_0's binary_logloss: 0.596838
[1400]	valid_0's auc: 0.689791	valid_0's binary_logloss: 0.59680[...]<jupyter_text>Crazy Method - 2달 미뤄서 훈련시켜보기 2<jupyter_code>temp = data[(data.Date_YMD >= '2018-07-26') & (data.Date_YMD <= '2018-11-25')]
del temp['Date']
gc.collect()
train = temp[~temp.HasDetections.isna()]
test = temp[temp.HasDetections.isna()]

del temp
gc.collect()
train = train[train.Date_YMD <= '2018-09-25']
test = test[test.Date_YMD >= '2018-09-26']

gc.collect()
# 2달 미루기
train.Date_YMD = train.Date_YMD + pd.offsets.MonthOffset(2)
train.reset_index(drop=True, inplace=True)
test.reset_index(drop=True, inplace=True)
train.shape
test.shape
del train['AvSigVersion'], test['AvSigVersion']
gc.collect()
print('Transform all features to category.\n')
for usecol in tqdm_notebook([col for col in train.columns if col not in ['HasDetections', 'MachineIdentifier']]):

    train[usecol] = train[usecol].astype('str')
    test[usecol] = test[usecol].astype('str')
    
    #Fit LabelEncoder
    le = LabelEncoder().fit(
            np.unique(train[usecol].unique().tolist()+
                      test[usecol].unique().tolist()))

    #At the end 0 will be used for dropped values
    train[usecol] = le.transform(train[usecol])+1
    test[usecol]  = le.transform(test[usecol])+1

    agg_tr = (train
              .groupby([usecol])
              .aggregate({'MachineIdentifier':'count'})
              .reset_index()
              .rename({'MachineIdentifier':'Train'}, axis=1))
    
    agg_te = (test
              .groupby([usecol])
              .aggregate({'MachineIdentifier':'count'})
              .reset_index()
              .rename({'MachineIdentifier':'Test'}, axis=1))

    agg = pd.merge(agg_tr, agg_te, on=usecol, how='outer').replace(np.nan, 0)

    #Select values with more than 1000 observations
    agg = agg[(agg['Train'] > 960)].reset_index(drop=True)
    agg['Total'] = agg['Train'] + agg['Test']
    #Drop unbalanced values
    agg = agg[(agg['Train'] / (agg['Total']) > 0.2) & (agg['Train'] / (agg['Total']) < 0.8)]
    agg[usecol+'Copy'] = agg[usecol]
        
    train[usecol] = (pd.merge(train[[usecol]], 
                              agg[[usecol, usecol+'Copy']], 
                              on=usecol, how='left')[usecol+'Copy']
                     .replace(np.nan, 0).astype('int').astype('category'))

    test[usecol]  = (pd.merge(test[[usecol]], 
                              agg[[usecol, usecol+'Copy']], 
                              on=usecol, how='left')[usecol+'Copy']
                     .replace(np.nan, 0).astype('int').astype('category'))

    del le, agg_tr, agg_te, agg, usecol
    gc.collect()
train.shape
y_train = np.array(train['HasDetections'])
train_ids = train.index
test_ids  = test.index
del train['HasDetections'], train['MachineIdentifier'], test['MachineIdentifier'], test['HasDetections']
gc.collect()
print("If you don't want use Sparse Matrix choose Kernel Version 2 to get simple solution.\n")

print('--------------------------------------------------------------------------------------------------------')
print('Transform Data to Sparse Matrix.')
print('Sparse Matrix can be used to fit a lot of models, eg. XGBoost, LightGBM, Random Forest, K-Means and etc.')
print('To concatenate Sparse Matrices by column use hstack()')
print('Read more about Sparse Matrix https://docs.scipy.org/doc/scipy/reference/sparse.html')
print('Good Luck!')
print('--------------------------------------------------------------------------------------------------------')

#Fit OneHotEncoder
ohe = OneHotEncoder(categories='auto', sparse=True, dtype='uint8').fit(train.append(test))
#Transform data using small groups to reduce memory usage
m = 100000
train = vstack([ohe.transform(train[i*m:(i+1)*m]) for i in range(train.shape[0] // m + 1)])
test  = vstack([ohe.transform(test[i*m:(i+1)*m])  for i in range(test.shape[0] // m +  1)])
train.shape
test.shape
save_npz('./data_temp/train.npz', train, compressed=True)
save_npz('./data_temp/test.npz',  test,  compressed=True)
del ohe, train, test
gc.collect()
train = load_npz('./data_temp/train.npz')
test = load_npz('./data_temp/test.npz')
test = csr_matrix(test, dtype='float32')
train.shape
test.shape
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
skf.get_n_splits(train_ids, y_train)

lgb_test_result  = np.zeros(test_ids.shape[0])
lgb_train_result = np.zeros(train_ids.shape[0])
counter = 0
print('\nLightGBM\n')

for train_index, test_index in skf.split(train_ids, y_train):
    
    print('Fold {}\n'.format(counter + 1))
    
    X_fit = vstack([train[train_index[i*m:(i+1)*m]] for i in range(train_index.shape[0] // m + 1)])
    X_val = vstack([train[test_index[i*m:(i+1)*m]]  for i in range(test_index.shape[0] //  m + 1)])
    X_fit, X_val = csr_matrix(X_fit, dtype='float32'), csr_matrix(X_val, dtype='float32')
    y_fit, y_val = y_train[train_index], y_train[test_index]
    
    gc.collect()

    lgb_model = lgb.LGBMClassifier(max_depth=-1,
                                   n_estimators=30000,
                                   learning_rate=0.05,
                                   num_leaves=2**12-1,
                                   objective='binary', 
                                   boosting_type='gbdt',
                                   # overfitting handling
                                   # max_bin=120,
                                   # lambda_l1=6,
                                   # lambda_l2=2,
                                   save_binary=True,
                                   feature_fraction=0.8,
                                   feature_fraction_seed=42,
                                   n_jobs=-1)
    
    print("fitting")
    lgb_model.fit(X_fit, y_fit, eval_metric='auc', 
                  eval_set=[(X_val, y_val)], 
                  verbose=100, early_stopping_rounds=100)
    
    del X_fit, X_val, y_fit, y_val, train_index, test_index
    gc.collect()
    
    print("predicting")
    lgb_test_result += lgb_model.predict_proba(test)[:,1]
    counter += 1
    
    gc.collect()
pd.DataFrame(lgb_test_result/3).to_csv('./data_temp/temp_11.csv')
data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15669911914696338, 'fraction_numerical': 0.050268229684085036, 'mean_word_length': 2.5666706035195466, 'pattern_counts': {'":': 0, '<': 104, '<?xml version=': 0, '>': 106, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 85, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '288033', 'license': 'MIT', 'n_tokens_mistral': 11182, 'n_tokens_neox': 10103, 'n_words': 1527, 'path': 'Microsoft Malware Prediction/code/preprocessing/03-5 FE - Time Related New CV - 2018-07-24 ~.ipynb', 'repo_name': 'choco9966/kaggle', 'size': '30198'}