<jupyter_start><jupyter_text>Spliting the data into features and target for isFraud=0 and isFraud=1<jupyter_code>data_isFraud_1_indexes=list(data[data['isFraud']==1].index)
data_isFraud_0_indexes=list(data[data['isFraud']==0].index)
data_isFraud_1=data.iloc[data_isFraud_1_indexes , : ]
data_isFraud_0=data.iloc[data_isFraud_0_indexes , : ]

target_isFraud_1=data_isFraud_1['isFraud']
print('target_isFraurd_1 shape: ',target_isFraud_1.shape)

features_isFraud_1=data_isFraud_1.drop(['isFraud'],axis=1)
print('features_isFraud_1 shape: ',features_isFraud_1.shape)

target_isFraud_0=data_isFraud_0['isFraud']
print('target_isFraurd_0 shape: ',target_isFraud_0.shape)

features_isFraud_0=data_isFraud_0.drop(['isFraud'],axis=1)
print('features_isFraud_0 shape: ',features_isFraud_0.shape)
features_25_0=features_isFraud_0[list(df_MI['Column'].head(25))]
features_50_0=features_isFraud_0[list(df_MI['Column'].head(50))]
features_75_0=features_isFraud_0[list(df_MI['Column'].head(75))]
features_100_0=features_isFraud_0[list(df_MI['Column'].head(100))]
features_150_0=features_isFraud_0[list(df_MI['Column'].head(150))]
features_200_0=features_isFraud_0[list(df_MI['Column'].head(200))]
features_25_0.shape
features_25_1=features_isFraud_1[list(df_MI['Column'].head(25))]
features_50_1=features_isFraud_1[list(df_MI['Column'].head(50))]
features_75_1=features_isFraud_1[list(df_MI['Column'].head(75))]
features_100_1=features_isFraud_1[list(df_MI['Column'].head(100))]
features_150_1=features_isFraud_1[list(df_MI['Column'].head(150))]
features_200_1=features_isFraud_1[list(df_MI['Column'].head(200))]
features_25_1.shape
from sklearn.model_selection import KFold
def get_kfolds(splits,features,target):
  kf = KFold(n_splits=splits)

  X=features
  Y=target

  X_train_base=[]
  X_test_base=[]
  Y_train_base=[]
  Y_test_base=[]

  for train_index, test_index in kf.split(X):
      #print('train_index', train_index)
      #print('test_index', test_index)
      X_train, X_test = X.iloc[train_index,:], X.iloc[test_index,:]
      Y_train, Y_test = Y.iloc[train_index], Y.iloc[test_index]
      X_train_base.append(X_train)
      X_test_base.append(X_test)
      Y_train_base.append(Y_train)
      Y_test_base.append(Y_test)
  
  return(X_train_base,X_test_base,Y_train_base,Y_test_base)
  


MI_TOP_0=[features_25_0,features_50_0,features_75_0,features_100_0,features_150_0,features_200_0]

MI_TOP_1=[features_25_1,features_50_1,features_75_1,features_100_1,features_150_1,features_200_1]

train_features=[]
train_target=[]
test_features=[]
test_target=[]
for i in range(len(MI_TOP_0)):
  X_train_0,X_test_0,Y_train_0,Y_test_0=get_kfolds(2,MI_TOP_0[i],target_isFraud_0)
  X_train_1,X_test_1,Y_train_1,Y_test_1=get_kfolds(2,MI_TOP_1[i],target_isFraud_1)
  for j in range(len(X_train_0)):
    tr_features = pd.concat([pd.DataFrame(X_train_0[j]), pd.DataFrame(X_train_1[j])], axis=0)
    tr_target = pd.concat([pd.DataFrame(Y_train_0[j]), pd.DataFrame(Y_train_1[j])], axis=0)
    te_features = pd.concat([pd.DataFrame(X_test_0[j]), pd.DataFrame(X_test_1[j])], axis=0)
    te_target = pd.concat([pd.DataFrame(Y_test_0[j]), pd.DataFrame(Y_test_1[j])], axis=0)
    X = np.array(tr_features)
    y = np.array(tr_target)
    sm = SMOTE(random_state=2)
    X_res, y_res = sm.fit_sample(X, y)
    #print("Number transactions X dataset: ", X.shape)
    #print("Number transactions y dataset: ", y.shape)
    #print("Number transactions X_res dataset: ", X_res.shape)
    #print("Number transactions y_res dataset: ", y_res.shape)
    #print("Before OverSampling, counts of label '1': {}".format(sum(y==1)))
    #print("Before OverSampling, counts of label '0': {} \n".format(sum(y==0)))
    #print("After OverSampling, counts of label '1': {}".format(sum(y_res==1)))
    #print("After OverSampling, counts of label '0': {} \n".format(sum(y_res==0)))
    train_features.append(X_res)
    train_target.append(y_res)
    X_te = np.array(te_features)
    y_te = np.array(te_target)

    
    test_features.append(X_te)
    test_target.append(y_te)

model = LogisticRegression()  # Choose a model here
for i in range(len(train_features)):
   model.fit(train_features[i], train_target[i] )
   y_pred = model.predict(test_features[i])
   print(f'Number of TOP MI: {train_features[i].shape[1]}')
   print(f'Number fold: {i}')

   f_score=f1_score(test_target[i], y_pred)
   cm = metrics.confusion_matrix(test_target[i], y_pred)
   print(cm)

   print(f'f-score: {f_score}')

   print(classification_report(test_target[i], y_pred))<jupyter_output>/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.
  FutureWarning)<jupyter_text>Testing on the real Test Data<jupyter_code>real_test=pd.read_csv('gdrive/My Drive/Colab Notebooks/Predictive Modeling/DATA/3. fraud_remove_nulls_fill_random_distribution/test-fraud_remove_nulls_fill_random_distribution.csv',compression='gzip')
from pandas.api.types import CategoricalDtype

# M1-M9: T/F/Blank
real_test['M1']=real_test['M1'].astype(CategoricalDtype(categories=['F','T']))
real_test['M1'] = real_test.M1.cat.codes
real_test['M2']=real_test['M2'].astype(CategoricalDtype(categories=['F','T']))
real_test['M2'] = real_test.M2.cat.codes
real_test['M3']=real_test['M3'].astype(CategoricalDtype(categories=['F','T']))
real_test['M3'] = real_test.M3.cat.codes

real_test['M4']=real_test['M4'].astype(CategoricalDtype(categories=['M0','M1','M2']))
real_test['M4'] = real_test.M4.cat.codes

real_test['M5']=real_test['M5'].astype(CategoricalDtype(categories=['F','T']))
real_test['M5'] = real_test.M5.cat.codes
real_test['M6']=real_test['M6'].astype(CategoricalDtype(categories=['F','T']))
real_test['M6'] = real_test.M6.cat.codes
real_test['M7']=real_test['M7'].astype(CategoricalDtype(categories=['F','T']))
real_test['M7'] = real_test.M7.cat.codes
real_test['M8']=real_test['M8'].astype(CategoricalDtype(categories=['F','T']))
real_test['M8'] = real_test.M8.cat.codes
real_test['M9']=real_test['M9'].astype(CategoricalDtype(categories=['F','T']))
real_test['M9'] = real_test.M9.cat.codes

real_test['ProductCD']=real_test['ProductCD'].astype(CategoricalDtype())
real_test['ProductCD'] = real_test.ProductCD.cat.codes

real_test['device_name']=real_test['device_name'].astype(CategoricalDtype())
real_test['device_name'] = real_test.device_name.cat.codes

real_test['DeviceType']=real_test['DeviceType'].astype(CategoricalDtype())
real_test['DeviceType'] = real_test.DeviceType.cat.codes

real_test['addr1']=real_test['addr1'].astype(CategoricalDtype())
real_test['addr1'] = real_test.addr1.cat.codes
real_test['addr2']=real_test['addr2'].astype(CategoricalDtype())
real_test['addr2'] = real_test.addr2.cat.codes

real_test['card1']=real_test['card1'].astype(CategoricalDtype())
real_test['card1'] = real_test.card1.cat.codes
real_test['card2']=real_test['card2'].astype(CategoricalDtype())
real_test['card2'] = real_test.card2.cat.codes
real_test['card3']=real_test['card3'].astype(CategoricalDtype())
real_test['card3'] = real_test.card3.cat.codes
real_test['card4']=real_test['card4'].astype(CategoricalDtype())
real_test['card4'] = real_test.card4.cat.codes
real_test['card5']=real_test['card5'].astype(CategoricalDtype())
real_test['card5'] = real_test.card5.cat.codes
real_test['card6']=real_test['card6'].astype(CategoricalDtype())
real_test['card6'] = real_test.card6.cat.codes

real_test['P_emaildomain_bin']=real_test['P_emaildomain_bin'].astype(CategoricalDtype())
real_test['P_emaildomain_bin'] = real_test.P_emaildomain_bin.cat.codes
real_test['R_emaildomain_bin']=real_test['R_emaildomain_bin'].astype(CategoricalDtype())
real_test['R_emaildomain_bin'] = real_test.R_emaildomain_bin.cat.codes
real_test['P_emaildomain_suffix']=real_test['P_emaildomain_suffix'].astype(CategoricalDtype())
real_test['P_emaildomain_suffix'] = real_test.P_emaildomain_suffix.cat.codes
real_test['R_emaildomain_suffix']=real_test['R_emaildomain_suffix'].astype(CategoricalDtype())
real_test['R_emaildomain_suffix'] = real_test.R_emaildomain_suffix.cat.codes

real_test['_Month']=real_test['_Month'].astype(CategoricalDtype())
real_test['_Month'] = real_test._Month.cat.codes
real_test['_Weekdays']=real_test['_Weekdays'].astype(CategoricalDtype())
real_test['_Weekdays'] = real_test._Weekdays.cat.codes
real_test['_Hours']=real_test['_Hours'].astype(CategoricalDtype())
real_test['_Hours'] = real_test._Hours.cat.codes
real_test['_Days']=real_test['_Days'].astype(CategoricalDtype())
real_test['_Days'] = real_test._Days.cat.codes

real_test['id_12']=real_test['id_12'].astype(CategoricalDtype())
real_test['id_12'] = real_test.id_12.cat.codes 
real_test['id_13']=real_test['id_13'].astype(CategoricalDtype())
real_test['id_13'] = real_test.id_13.cat.codes 
real_test['id_15']=real_test['id_15'].astype(CategoricalDtype())
real_test['id_15'] = real_test.id_15.cat.codes 
real_test['id_16']=real_test['id_16'].astype(CategoricalDtype())
real_test['id_16'] = real_test.id_16.cat.codes 
real_test['id_17']=real_test['id_17'].astype(CategoricalDtype())
real_test['id_17'] = real_test.id_17.cat.codes 
real_test['id_19']=real_test['id_19'].astype(CategoricalDtype())
real_test['id_19'] = real_test.id_19.cat.codes 
real_test['id_20']=real_test['id_20'].astype(CategoricalDtype())
real_test['id_20'] = real_test.id_20.cat.codes
real_test['id_28']=real_test['id_28'].astype(CategoricalDtype())
real_test['id_28'] = real_test.id_28.cat.codes
real_test['id_29']=real_test['id_29'].astype(CategoricalDtype())
real_test['id_29'] = real_test.id_29.cat.codes
real_test['id_31']=real_test['id_31'].astype(CategoricalDtype())
real_test['id_31'] = real_test.id_31.cat.codes
real_test['id_35']=real_test['id_35'].astype(CategoricalDtype())
real_test['id_35'] = real_test.id_35.cat.codes
real_test['id_36']=real_test['id_36'].astype(CategoricalDtype())
real_test['id_36'] = real_test.id_36.cat.codes
real_test['id_37']=real_test['id_37'].astype(CategoricalDtype())
real_test['id_37'] = real_test.id_37.cat.codes
real_test['id_38']=real_test['id_38'].astype(CategoricalDtype())
real_test['id_38'] = real_test.id_38.cat.codes


features_select=data[list(df_MI['Column'].head(150))]
test_features_select=real_test[list(df_MI['Column'].head(150))]
y_train=np.array(list(data['isFraud']))# .reshape(-1, 1)
X_train = np.array(features_select)
sm = SMOTE(random_state=2)
#sm = SMOTE(random_state=12, ratio = 1.0)
x_train_res, y_train_res = sm.fit_sample(X_train, y_train)
print("After OverSampling, counts of label '1': {}".format(sum(y_train_res==1)))
print("After OverSampling, counts of label '0': {} \n".format(sum(y_train_res==0)))
y_real_test=np.array(list(real_test['isFraud'])) #.reshape(-1, 1)
X_real_test = np.array(test_features_select)
model = LogisticRegression()  # Choose a model here

model.fit(x_train_res, y_train_res )
y_pred = model.predict(X_real_test)
f_score=f1_score(y_real_test, y_pred)
print(f'f-score: {f_score}')

cm = metrics.confusion_matrix(y_real_test, y_pred)
print(cm)




print(classification_report(y_real_test, y_pred))<jupyter_output>precision    recall  f1-score   support

           0       0.99      0.82      0.89    170964
           1       0.12      0.67      0.20      6199

    accuracy                           0.81    177163
   macro avg       0.55      0.74      0.55    177163
weighted avg       0.96      0.81      0.87    177163
{'fraction_non_alphanumeric': 0.16647662485746864, 'fraction_numerical': 0.042540128058942195, 'mean_word_length': 8.307755102040817, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '570617', 'license': 'MIT', 'n_tokens_mistral': 4845, 'n_tokens_neox': 4605, 'n_words': 453, 'path': 'Imbalance Model CV and testing.ipynb', 'repo_name': 'dinbav/Fraud', 'size': '11401'}