<jupyter_start><jupyter_text>leggiamo l'intero LARGE dataset, augmented, dal file in formato Feather<jupyter_code>cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']

dtypes = {
        'ip'            : 'uint32',
        'app'           : 'uint16',
        'device'        : 'uint16',
        'os'            : 'uint16',
        'channel'       : 'uint16',
        'is_attributed' : 'uint8',
        }
# adding new features

def add_new_features(df):
    df['click_time'] = pd.to_datetime(df['click_time'])
    df['hour'] = pd.to_datetime(df.click_time).dt.hour.astype('uint8')
    df['day'] = pd.to_datetime(df.click_time).dt.day.astype('uint8')
    
    print('group by : ip_app_channel_var_day')
    gp = df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app', 'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})
    df = df.merge(gp, on=['ip','app', 'channel'], how='left')
    del gp
    gc.collect()
    
    print('group by : ip_day_hour_count_chl')
    gp = df[['ip','day','hour','channel']].groupby(by=['ip','day','hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'})
    df = df.merge(gp, on=['ip','day','hour'], how='left')
    del gp
    gc.collect()
    
    print('group by : ip_app_count_chl')
    gp = df[['ip','app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})
    df = df.merge(gp, on=['ip','app'], how='left')
    del gp
    gc.collect()
    
    print('group by : ip_app_os_count_chl')
    gp = df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})
    df = df.merge(gp, on=['ip','app', 'os'], how='left')
    del gp
    gc.collect()
    
    print('group by : ip_day_chl_var_hour')
    gp = df[['ip','day','hour','channel']].groupby(by=['ip','day','channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'qty_var'})
    df = df.merge(gp, on=['ip','day','channel'], how='left')
    del gp
    gc.collect()
    
    print('group by : ip_app_os_var_hour')
    gp = df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', 'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})
    df = df.merge(gp, on=['ip','app', 'os'], how='left')
    del gp
    gc.collect()
    
    print('group by : ip_app_chl_mean_hour')
    gp = df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app', 'channel'])[['hour']].mean().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_mean_hour'})
    df = df.merge(gp, on=['ip','app', 'channel'], how='left')
    del gp
    gc.collect()
    
    return df
# RESTART POINT

orig_train = pd.read_feather("train_augmented.fea")
orig_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 184903890 entries, 0 to 184903889
Data columns (total 16 columns):
 #   Column                    Dtype         
---  ------                    -----         
 0   ip                        uint32        
 1   app                       uint16        
 2   device                    uint16        
 3   os                        uint16        
 4   channel                   uint16        
 5   click_time                datetime64[ns]
 6   is_attributed             uint8         
 7   hour                      uint8         
 8   day                       uint8         
 9   ip_app_channel_var_day    float64       
 10  qty                       uint16        
 11  ip_app_count              uint16        
 12  ip_app_os_count           uint16        
 13  qty_var                   float64       
 14  ip_app_os_var             float64       
 15  ip_app_channel_mean_hour  float64       
dtypes: datetime64[ns](1), float64(4), uint16(7), uint3[...]<jupyter_text>* il dataset è estremamente sbilanciato... andrebbe fatto downsampling della classe 0* il downsampling lo faccio dopo lo split train validation<jupyter_code>TARGET = 'is_attributed'

# train, validation split
TRAIN_FRAC = 0.98

N_TRAIN = int(orig_train.shape[0] * TRAIN_FRAC)

# shuffle
orig_train = orig_train.sample(frac=1.)

df_train = orig_train[:N_TRAIN]
df_valid = orig_train[N_TRAIN:]

print('Original training samples:', df_train.shape[0])
print('Num. validation samples:', df_valid.shape[0])
orig_train.columns
# rebalance

# removed ip because it is high card
feat = ['ip', 'app', 'device', 'os', 'channel',
       'hour', 'day', 'ip_app_channel_var_day', 'qty', 'ip_app_count',
       'qty_var', 'ip_app_os_count', 'ip_app_os_var',
       'ip_app_channel_mean_hour']
cat_feat = ['app', 'device', 'os', 'channel']

x_train = df_train[feat]
y_train = df_train[TARGET]

x_valid = df_valid[feat]
y_valid = df_valid[TARGET]

rus = RandomUnderSampler(random_state=0)

x_train_resampled, y_train_resampled = rus.fit_resample(x_train, y_train)
%%time

model = lgb.LGBMClassifier(objective='binary', seed=4321, n_jobs=-1, n_estimators=1000, num_leaves = 30, max_depth=15, early_stopping_rounds=10)

best_model = model.fit(x_train_resampled, y_train_resampled, eval_set=[(x_valid, y_valid)], verbose=1, feature_name=feat, categorical_feature=cat_feat, eval_metric = 'auc')
lgb.plot_metric(model, metric='auc', figsize=(9, 6), title='validation AUC');
lgb.plot_importance(model, figsize=(9, 6), max_num_features=15)<jupyter_output><empty_output><jupyter_text>Predictions on test set<jupyter_code># make predictions on test set
orig_test = pd.read_csv('test.csv', dtype=dtypes)

# do feature engineering
orig_test = add_new_features(orig_test)
orig_test.head()
x_test = orig_test[feat].values

y_probs = best_model.predict_proba(x_test)

y_probs_pos = y_probs[:, 1] 

dict_sub = {}

dict_sub['click_id'] = orig_test['click_id'].values
dict_sub['is_attributed'] = y_probs_pos

df_sub = pd.DataFrame(dict_sub)

SUB_FILE = 'submission5.csv'

df_sub.to_csv(SUB_FILE, index=None)

# !kaggle competitions submit -c talkingdata-adtracking-fraud-detection -f $SUB_FILE -m "third submission"<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.171059093141267, 'fraction_numerical': 0.020952692748403993, 'mean_word_length': 3.3674052894924946, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '217163', 'license': 'MIT', 'n_tokens_mistral': 2308, 'n_tokens_neox': 2196, 'n_words': 396, 'path': 'model_one.ipynb', 'repo_name': 'luigisaetta/anomaly-large-data', 'size': '6109'}