<jupyter_start><jupyter_text>def base_feat(df):    df['time'] = df['time'].astype(str)    df['sec'] = df['time'].apply(lambda x: x.split(".")[-1])    df['min'] = df['time'].apply(lambda x: x.split(".")[0])    df['min'] = df['min'].astype(int)*60    df['sec'] = df['sec'].astype(int)    df['time'] = df['min']+tr['sec']        dfg = df.groupby(['game_id','player'])['time'].agg(['sum','mean','std','count','last'])    dfg = dfg.reset_index()        p0 = dfg[dfg['player']==0]    p1 = dfg[dfg['player']==1]        p0_cols = ["p0_"+x for x in p0.columns[2:]]    p1_cols = ["p1_"+x for x in p1.columns[2:]]        p0.columns = ['game_id','player']+p0_cols    p1.columns = ['game_id','player']+p1_cols        new_df = p0.drop('player',axis=1).merge(p1.drop('player',axis=1), on='game_id',how='left')     new_df['game_duration'] = df.groupby(['game_id'])['time'].max()    del p0,p1            spec = df.groupby(['game_id','player'])['species'].agg(['max'])    spec = spec.reset_index()        p0_spec = spec[spec['player']==0]    p1_spec = spec[spec['player']==1]    p0_spec.drop(['player'], axis=1, inplace=True)    p1_spec.drop(['player'], axis=1, inplace=True)    p0_spec.columns = ['game_id','p0_species']    p1_spec.columns = ['game_id','p1_species']        new_df = new_df.merge(p0_spec, on='game_id', how='left')    new_df = new_df.merge(p1_spec, on='game_id', how='left')    new_df['p0_species'] = new_df['p0_species'].astype('category')    new_df['p1_species'] = new_df['p1_species'].astype('category')    return new_df %%timetr_set = base_feat(tr)te_set = base_feat(te)<jupyter_code>def make_timespan(df):

    df['time_span'] = 0
    df['time_span'] = np.where(df['time']<1, 0, 
                                  np.where(df['time']<2, 1, 
                                          np.where(df['time']<3, 2,
                                                  np.where(df['time']<4, 3,
                                                          np.where(df['time']<5, 4,
                                                                  np.where(df['time']<6, 5,
                                                                          np.where(df['time']<7, 6,
                                                                                  np.where(df['time']<8, 7,
                                                                                          np.where(df['time']<9, 8,
                                                                                                   np.where(df['time']<10, 9, 10))))))))))
    return df
tr = make_timespan(tr)
te = make_timespan(te)
tr.head()
def time_n_event(df):
    new_p0 = pd.DataFrame({"game_id":df['game_id'].unique()})
    new_p1 = pd.DataFrame({"game_id":df['game_id'].unique()})
    
    for tsp in tqdm(range(11)):
        p0 = df[(df['player']==0)&(df['time_span']==tsp)]
        tmp = p0.groupby(['game_id','time_span'])['event'].value_counts(dropna=False, normalize=True).unstack()
        tmp.columns= ['TIME_{}_EVENT_{}_p0'.format(tsp, eve_name) for eve_name in tmp.columns]
        tmp.reset_index(inplace=True)
        tmp.fillna(0, inplace=True)
        tmp.drop(['time_span'],axis=1,inplace=True)

        new_p0 = new_p0.merge(tmp, on=['game_id'],how='left')
        del tmp;gc.collect()
        
    for tsp in tqdm(range(11)):
        p1 = df[(df['player']==1)&(df['time_span']==tsp)]
        tmp = p1.groupby(['game_id','time_span'])['event'].value_counts(dropna=False, normalize=True).unstack()
        tmp.columns= ['TIME_{}_EVENT_{}_p1'.format(tsp, eve_name) for eve_name in tmp.columns]
        tmp.reset_index(inplace=True)
        tmp.fillna(0, inplace=True)
        tmp.drop(['time_span'],axis=1,inplace=True)

        new_p1 = new_p1.merge(tmp, on=['game_id'],how='left')
        del tmp;gc.collect()
    
    new_df = new_p0.merge(new_p1, on=['game_id'], how='left')
    
    return new_df    
tr_time_eve = time_n_event(tr)
tr_time_eve.head()
tr_time_eve.fillna(0, inplace=True)
tr_time_eve.head()
te_time_eve = time_n_event(te)
te_time_eve.fillna(0, inplace=True)
te_time_eve.head()<jupyter_output><empty_output><jupyter_text>Save feature files<jupyter_code># tr_time_eve.to_feather("../dacon_sc/input/tr_time_eve.feather")
# te_time_eve.to_feather("../dacon_sc/input/te_time_eve.feather")
tr_y = tr.groupby(['game_id'])['winner'].max()
tr_y = tr_y.reset_index()
train_Y = tr_y['winner']
tr_y.head()
def diff_all(df):
    p0_cols = df[df.filter(regex='p0').columns.tolist()].select_dtypes(exclude=[object,'category']).columns.tolist()
    p1_cols = df[df.filter(regex='p1').columns.tolist()].select_dtypes(exclude=[object,'category']).columns.tolist()
    
    for a, b in tqdm(zip(p0_cols, p1_cols), total=len(p0_cols)):
        df["{}_{}_diff".format(a,b)] = df[a] - df[b]
    
    return df
tr_time_eve = diff_all(tr_time_eve)
te_time_eve = diff_all(te_time_eve)
import time
from sklearn import metrics
from operator import itemgetter
import lightgbm as lgb
from sklearn.model_selection import KFold

def LGB_KFOLD_BINA(n_fold, train_X, test_X, metric, lr, num_leaves, max_depth):
    
    folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)

    oof_lgb = np.zeros(len(train_X))
    predictions = np.zeros(len(test_X))
    feature_importance_df = pd.DataFrame()
    cv_score_df = []

    # Model parameters
    lgb_params = {'num_leaves': num_leaves,
                 'min_data_in_leaf': 20, 
                 'objective':'binary',
                 'max_depth': max_depth,
                 'learning_rate': lr,
                 "boosting": "gbdt",
                 "feature_fraction": 0.3,
                 "bagging_freq": 1,
                 "bagging_fraction": 0.7,
                 "bagging_seed": 42,
                 "metric": metric,
                 "lambda_l1": 0.0,
                 "verbosity": 300,
                 "nthread": -1,
                 "random_state": 42}

    model_start = time.time()

    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_X)):
        strLog = "fold {}".format(fold_)
        print(strLog+ "-" * 50)

        X_tr, X_val = train_X.iloc[trn_idx], train_X.iloc[val_idx]
        y_tr, y_val = train_Y.iloc[trn_idx], train_Y.iloc[val_idx]

        model = lgb.LGBMClassifier(**lgb_params, n_estimators = 200000, n_jobs = -1)
        model.fit(X_tr, 
                  y_tr, 
                  eval_set=[(X_tr, y_tr), (X_val, y_val)], 
                  eval_metric=metric,
                  verbose=300, 
                  early_stopping_rounds=200)
        oof_lgb[val_idx] = model.predict_proba(X_val, num_iteration=model.best_iteration_)[:,1]

        cv_score_df.append(model.best_score_)

        #feature importance
        fold_importance_df = pd.DataFrame()
        fold_importance_df["Feature"] = train_X.columns
        fold_importance_df["importance"] = model.feature_importances_[:len(train_X.columns)]
        fold_importance_df["fold"] = fold_ + 1
        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
        #predictions
        predictions += model.predict_proba(test_X, num_iteration=model.best_iteration_)[:,1] / folds.n_splits

    cv_score_df = pd.DataFrame.from_dict(cv_score_df)
    cv_score_df = cv_score_df.valid_1.tolist()
    cv_score_df = list(map(itemgetter(metric),cv_score_df))

    print("-" * 50)
    #print("SF   RMSE = {}".format(oof_score))
    print("Mean "+ metric + " = {}".format(np.mean(cv_score_df)))
    print("Std "+ metric + " = {}".format(np.std(cv_score_df)))
#     lgb.plot_metric(model, metric=metric, title='auc plot', xlabel='Iterations', ylabel='auto', figsize=(10,8), grid=False)

    model_end = time.time()
    model_elapsed = model_end - model_start
    print('Model elapsed {0:0.2f}'.format(model_elapsed/60), "minutes.")
    cols = (feature_importance_df[["Feature", "importance"]]
        .groupby("Feature")
        .mean()
        .sort_values(by="importance", ascending=False)[:].index)
    best_features = feature_importance_df.loc[feature_importance_df.Feature.isin(cols)]
    pd.set_option('display.max_rows', 500)
    best_features = best_features.drop(['fold'],axis=1)
    best_features = best_features.groupby(['Feature'], as_index = False).mean()
    best_features['Feature Rank'] = best_features['importance'].rank(ascending=0)
    best_features = best_features.sort_values('Feature Rank', ascending = True)
    print(best_features.loc[best_features['importance']!=0].head(100))

    return predictions, oof_lgb, np.mean(cv_score_df), np.std(cv_score_df), best_features
tr_time_eve.dtypes.value_counts()
#early stopping 200
pred, oof, cv, cv_std, best_feat = LGB_KFOLD_BINA(5, tr_time_eve.drop('game_id',axis=1), te_time_eve.drop('game_id',axis=1), "auc", lr=0.003, num_leaves=700, max_depth=-1)
#early stopping 200
pred, oof, cv, cv_std, best_feat = LGB_KFOLD_BINA(5, tr_time_eve.drop('game_id',axis=1), te_time_eve.drop('game_id',axis=1), "auc", lr=0.003, num_leaves=700, max_depth=-1)
%%time
# day_list = ['Friday','Monday','Saturday','Sunday','Thursday','Tuesday','Wednesday']
tr_p0 = tr[tr['player']==0].groupby(['game_id','player'])['event'].value_counts(dropna=False, normalize=True).unstack()
tr_p0 = tr_p0.reset_index()
tr_p1 = tr[tr['player']==1].groupby(['game_id','player'])['event'].value_counts(dropna=False, normalize=True).unstack()
tr_p1 = tr_p1.reset_index()
te_p0 = te[te['player']==0].groupby(['game_id','player'])['event'].value_counts(dropna=False, normalize=True).unstack()
te_p0 = te_p0.reset_index()
te_p1 = te[te['player']==1].groupby(['game_id','player'])['event'].value_counts(dropna=False, normalize=True).unstack()
te_p1 = te_p1.reset_index()
tr_p0.columns.name =""
tr_p1.columns.name =""
te_p0.columns.name =""
te_p1.columns.name =""
p0_cols = ["p0_"+x for x in tr_p0.columns[2:]]
p1_cols = ["p1_"+x for x in tr_p1.columns[2:]]
tr_p0.columns = ['game_id','player']+p0_cols
tr_p1.columns = ['game_id','player']+p1_cols
te_p0.columns = ['game_id','player']+p0_cols
te_p1.columns = ['game_id','player']+p1_cols
print(tr_p0.shape)
tr_p0.head()
tr_p1.head()
te_p1.head()
new_tr = tr_p0.drop('player',axis=1).merge(tr_p1.drop('player',axis=1), on='game_id', how='left')
new_te = te_p0.drop('player',axis=1).merge(te_p1.drop('player',axis=1), on='game_id', how='left')
new_tr.head()
tr_set.head()
new_tr.head()
new_tr.fillna(0, inplace=True)
new_te.fillna(0, inplace=True)
%%time
new_tr = new_tr.merge(tr_set, on='game_id',how='left')
new_te = new_te.merge(te_set, on='game_id',how='left')
new_tr.dtypes
new_tr.head()
new_tr.to_feather("../dacon_sc/input/new_tr.feather")
new_te.to_feather("../dacon_sc/input/new_te.feather")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17747087561067268, 'fraction_numerical': 0.02236001503194288, 'mean_word_length': 3.297537343560759, 'pattern_counts': {'":': 12, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '116109', 'license': 'Apache-2.0', 'n_tokens_mistral': 4048, 'n_tokens_neox': 3902, 'n_words': 525, 'path': 'time_n_event_tsp.ipynb', 'repo_name': 'marquis08/dacon_sc', 'size': '10644'}