<jupyter_start><jupyter_text>load package<jupyter_code>import pandas as pd
import pickle
import copy
pd.set_option('display.max_columns', 50)<jupyter_output><empty_output><jupyter_text>load training dataset<jupyter_code>train_path ='./data/trainset-281-29.xlsx'
train_df = pd.read_excel(train_path)<jupyter_output><empty_output><jupyter_text>feature processing<jupyter_code>from data_processing import processing

p_conn =processing.processing()<jupyter_output><empty_output><jupyter_text>missing value<jupyter_code>missing_value =-999
missing_label ='missing'
for col in train_df.columns:
    train_df[col] =train_df[col].apply(lambda x : None if str.lower(str(x)) in ['none','non','nan'] else x)
train_df.fillna(-999,inplace=True)<jupyter_output><empty_output><jupyter_text>discrete features -more than 10<jupyter_code>discrete_cols_over_10 =['CARR_NAME','RGN_NAME','STATE_PRVNC_TXT','CUST_STATE']
risk_map =pickle.load(open(f"{model_dir}/risk_map.pkl", "rb"))
# reduce dimension with risk ratio
for idx,col in enumerate(discrete_cols_over_10):
    print(f"...{col}")
    target_col = f"{col}_bin_10_feature"
    value_map =risk_map[col]
    train_df[target_col] = train_df[col].apply(lambda x : value_map.get(x))<jupyter_output>...CARR_NAME
...RGN_NAME
...STATE_PRVNC_TXT
...CUST_STATE<jupyter_text>discrete features -less than 10    none treatment  datetime formate features<jupyter_code>ts_cols =['PWD_UPDT_TS','PH_NUM_UPDT_TS', 'TRAN_TS']<jupyter_output><empty_output><jupyter_text>operating hour<jupyter_code>for idx,col in enumerate(ts_cols):
    target_col = f"{col}_hour"
    train_df = p_conn.operate_hour(train_df,col)<jupyter_output><empty_output><jupyter_text>gap between updating operation, like passwaor or phone number,and transaction<jupyter_code>train_df['PWD_UPDT_TS_day'] = train_df.apply(lambda x : p_conn.timedelta_day(x.TRAN_TS,x.PWD_UPDT_TS),axis =1)    
train_df['PH_NUM_UPDT_TS_day'] = train_df.apply(lambda x : p_conn.timedelta_day(x.TRAN_TS,x.PH_NUM_UPDT_TS),axis =1)<jupyter_output><empty_output><jupyter_text>feature label to idx<jupyter_code>txt_value_idx_map =pickle.load(open(f'{model_dir}/txt_value_idx_map.pkl','rb'))
discrete_features =['CARR_NAME_bin_10_feature','PH_NUM_UPDT_TS_hour','RGN_NAME_bin_10_feature',
                   'STATE_PRVNC_TXT_bin_10_feature','PWD_UPDT_TS_hour','PH_NUM_UPDT_TS_day','DVC_TYPE_TXT']

continuous_features =['ACCT_PRE_TRAN_AVAIL_BAL','TRAN_AMT','OPEN_ACCT_CT','WF_dvc_age',]


filterd_features = discrete_features+continuous_features
for col in discrete_features:
    train_df[col] = train_df[col].apply(lambda x : txt_value_idx_map[x])<jupyter_output><empty_output><jupyter_text>model select & training<jupyter_code>from sklearn.model_selection import train_test_split

from sklearn.metrics import classification_report
label ='FRAUD_NONFRAUD'
label_map ={'Fraud':1,'Non-Fraud':0}


xgb_df =copy.deepcopy(train_df[filterd_features+[label]])

train_x,test_x = train_test_split(xgb_df,
                                 test_size=0.3,
                                 shuffle=True)


train_x =pd.concat([train_x,train_x[train_x['FRAUD_NONFRAUD']=='Fraud']])

train_x = train_x.sample(frac=1)

train_y =train_x.pop('FRAUD_NONFRAUD')
train_y =[label_map.get(i) for i in train_y ]


test_y =test_x.pop('FRAUD_NONFRAUD')
test_y =[label_map.get(i) for i in test_y ]<jupyter_output><empty_output><jupyter_text>random forest<jupyter_code>from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(max_depth=10, random_state =2021)
clf.fit(train_x, train_y)

# train dataset
predict_train_y = clf.predict(train_x)
predict_train_y =predict_train_y.tolist()
print(f"....random forest classification report---training dataset")
print('\n')
print(classification_report(y_true=train_y,y_pred=predict_train_y))

# test dataset
predict_y = clf.predict(test_x)
predict_y =predict_y.tolist()
print(f"....random forest classification report---test dataset")
print('\n')
print(classification_report(y_true=test_y,y_pred=predict_y))

print('++'*40)

print(f"feature importance")

print(dict(zip(filterd_features,clf.feature_importances_.tolist())))<jupyter_output>....random forest classification report---training dataset


              precision    recall  f1-score   support

           0       0.98      0.98      0.98      6869
           1       0.98      0.98      0.98      5862

    accuracy                           0.98     12731
   macro avg       0.98      0.98      0.98     12731
weighted avg       0.98      0.98      0.98     12731

....random forest classification report---test dataset


              precision    recall  f1-score   support

           0       0.98      0.96      0.97      2967
           1       0.91      0.95      0.93      1233

    accuracy                           0.96      4200
   macro avg       0.95      0.96      0.95      4200
weighted avg       0.96      0.96      0.96      4200

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
feature importance
{'CARR_NAME_bin_10_feature': 0.16296522285073528, 'PH_NUM_UPDT_TS_hour': 0.013030193444410981, 'RGN_NAME_bin_10_feature': 0.0604[...]<jupyter_text>save model -random forest<jupyter_code>pickle.dump(clf, open(f"{model_dir}/randomforest.pkl", "wb"))<jupyter_output><empty_output><jupyter_text>xgboost<jupyter_code>import xgboost as xgb
thres =0.5

bst =xgb.XGBClassifier(base_score=thres,
                       scale_pos_weight=0.7,
                       min_child_weight=10,
                       subsample =0.7,
                       colsample_bytree=0.7,
                      max_depth =8)

bst.fit(train_x.to_numpy(),train_y)
# training dataset
# predict_train_y = bst.predict(dtrain)
predict_train_y = bst.predict(train_x)

predict_train_y =predict_train_y.tolist()
predict_train_y =[1 if i >=thres else 0 for i in predict_train_y]
print(f"....xgboost classification report---training dataset")
print('\n')
print(classification_report(y_true=train_y,y_pred=predict_train_y))


predict_y_xgb = bst.predict(test_x)
predict_y_xgb =predict_y_xgb.tolist()
predict_y_xgb =[1 if i >=thres else 0 for i in predict_y_xgb]
print(f"....xgboost classification report---testing dataset")
print('\n')
print(classification_report(y_true=test_y,y_pred=predict_y_xgb))


print('++'*40)

print(f"feature importance")

print(dict(zip(filterd_features,bst.feature_importances_.tolist())))<jupyter_output>....xgboost classification report---training dataset


              precision    recall  f1-score   support

           0       0.99      0.99      0.99      6869
           1       0.99      0.98      0.99      5862

    accuracy                           0.99     12731
   macro avg       0.99      0.99      0.99     12731
weighted avg       0.99      0.99      0.99     12731

....xgboost classification report---testing dataset


              precision    recall  f1-score   support

           0       0.98      0.98      0.98      2967
           1       0.94      0.95      0.95      1233

    accuracy                           0.97      4200
   macro avg       0.96      0.96      0.96      4200
weighted avg       0.97      0.97      0.97      4200

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
feature importance
{'CARR_NAME_bin_10_feature': 0.32374516129493713, 'PH_NUM_UPDT_TS_hour': 0.02467169798910618, 'RGN_NAME_bin_10_feature': 0.05744619667530[...]<jupyter_text>save model- xgboost<jupyter_code>pickle.dump(bst, open(f"{model_dir}/bst.pkl", "wb"))<jupyter_output><empty_output><jupyter_text>DeepFM model<jupyter_code>import torch
from sklearn.preprocessing import MinMaxScaler

import numpy as np
import random
from deepctrmodels.deepfm import Deepfm
dense_features =['ACCT_PRE_TRAN_AVAIL_BAL', 'OPEN_ACCT_CT', 'TRAN_AMT', 'WF_dvc_age']
sparse_features =list(set(filterd_features)- set(dense_features))

feature_names =filterd_features
target = ['FRAUD_NONFRAUD']                     
seed = 1024
torch.manual_seed(seed)  
torch.cuda.manual_seed(seed) 
torch.cuda.manual_seed_all(seed)  
np.random.seed(seed)
random.seed(seed)
test_xx = copy.deepcopy(test_x)
mms = MinMaxScaler(feature_range=(-1, 1))
for df in [train_x,test_xx]:
    df[dense_features] = mms.fit_transform(df[dense_features])

# test_xx[dense_features] =mms.transform(test_xx[dense_features])

pickle.dump(mms, open(f"{model_dir}/max_min_scaler.pkl", "wb"))
# 2.count #unique features for each sparse field,and record dense feature field name
feat_sizes1={ feat:1 for feat in dense_features}
feat_sizes2 = {feat: max(txt_value_idx_map.values())+1 for feat in sparse_features}

# feat_sizes2 = {feat: len(train_df[feat].unique()) for feat in sparse_features}
feat_sizes={}
feat_sizes.update(feat_sizes1)
feat_sizes.update(feat_sizes2)
train_model_input = {name: train_x[name] for name in feature_names}
test_model_input =  {name: test_xx[name]  for name in feature_names}
# 4.Define Model,train,predict and evaluate
device = 'cpu'
use_cuda = True
if use_cuda and torch.cuda.is_available():
    print('cuda ready...')
    device = 'cuda:0'

model = Deepfm(feat_sizes ,
               sparse_feature_columns = sparse_features,
               sparse_shared_embedding_map ={},
               dense_feature_columns = dense_features,
               model_checkpoint_path =f"{model_dir}/deepfm.pt",
               dnn_hidden_units=[32,16] , dnn_dropout=0.5 , ebedding_size = 10,
               l2_reg_linear=1e-3, device=device)
model.fit(train_model_input, np.array(train_y) , 
          test_model_input , np.array(test_y)
          ,batch_size=256, epochs=150, verbose=1)<jupyter_output>cpu
Train on 12731 samples,  50 steps per epoch
epoch 0 train loss is 0.6394 train AUC is 0.8413
test LogLoss is 0.5526 test AUC is 0.9093
f1_score :0.8318708792849314
              precision    recall  f1-score   support

           0       0.89      0.87      0.88      2967
           1       0.71      0.73      0.72      1233

    accuracy                           0.83      4200
   macro avg       0.80      0.80      0.80      4200
weighted avg       0.83      0.83      0.83      4200

epoch 1 train loss is 0.4716 train AUC is 0.8884
test LogLoss is 0.4055 test AUC is 0.9062
f1_score :0.8311132520920345
epoch 2 train loss is 0.3784 train AUC is 0.9240
test LogLoss is 0.3501 test AUC is 0.9241
f1_score :0.8587791160759275
              precision    recall  f1-score   support

           0       0.91      0.88      0.90      2967
           1       0.74      0.79      0.77      1233

    accuracy                           0.86      4200
   macro avg       0.83      0.84      0.83    [...]<jupyter_text>model merge<jupyter_code>xgb_y_pred =bst.predict(test_x).tolist()
rf_y_pred =clf.predict(test_x).tolist()
df_y_pred =model.predict(test_xx,128)
df_y_pred =df_y_pred.reshape((1,-1)).tolist()[0]
test_x['xgb'] =xgb_y_pred
test_x['rf'] =rf_y_pred
test_x['df'] =[1 if i >0.55 else 0 for i in df_y_pred]
test_x['ground_truth'] = test_y
def label(value_list):
    label_sum = sum(value_list)
    label =0
    if label_sum >=2:
        label =1
    return label
test_x['merged_label'] = test_x.apply(lambda x : label([x.xgb,x.rf,x.df]),axis=1)
print(classification_report(test_x['ground_truth'],test_x['merged_label']))<jupyter_output>precision    recall  f1-score   support

           0       0.98      0.97      0.98      2967
           1       0.94      0.95      0.95      1233

    accuracy                           0.97      4200
   macro avg       0.96      0.96      0.96      4200
weighted avg       0.97      0.97      0.97      4200<jupyter_text>model redict result analysis<jupyter_code>test_x['correct_flag'] = test_x['ground_truth'] ==test_x['xgb']

test_x['trans_deposit_ratio'] = test_x['TRAN_AMT']/(test_x['ACCT_PRE_TRAN_AVAIL_BAL']+test_x['TRAN_AMT'])
test_x['trans_deposit_ratio_cls'] =pd.cut(test_x['trans_deposit_ratio'],bins=[i/10 for i in range(0,11)])
pd.pivot_table(data=test_x,index='trans_deposit_ratio_cls',columns='correct_flag',values ='xgb',aggfunc='count')
# groud truth ==non-fraud; predict result ==fraud
test_x[(test_x['correct_flag']==False)&
      (test_x['xgb']==1)]['trans_deposit_ratio_cls'].value_counts()
# groud truth == fraud; predict result ==non-fraud
test_x[(test_x['correct_flag']==False)&
      (test_x['xgb']==0)]['trans_deposit_ratio_cls'].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12495953382971836, 'fraction_numerical': 0.06191324053091615, 'mean_word_length': 3.44176851186197, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '604619', 'license': 'MIT', 'n_tokens_mistral': 4927, 'n_tokens_neox': 4500, 'n_words': 788, 'path': 'notebook/02 model-select-train.ipynb', 'repo_name': 'Intelligent-Rhino/finance_fraud_detect_deeplearning', 'size': '12356'}