<jupyter_start><jupyter_text>Preparation<jupyter_code>%%capture
!pip install catboost
import pandas as pd
import numpy as np
from sklearn.ensemble import IsolationForest
from sklearn.model_selection import train_test_split
from sklearn.metrics import (
    classification_report, 
    confusion_matrix,
    roc_auc_score,
    f1_score,
    accuracy_score
)
from sklearn.neighbors import LocalOutlierFactor

import matplotlib.pyplot as plt
import seaborn as sns

from xgboost import XGBClassifier
import xgboost as xgb

from catboost import Pool, CatBoostClassifier, cv

SEED = 2112
import warnings
warnings.filterwarnings('ignore')
df = pd.read_csv("/content/drive/MyDrive/applied_ai_enes_safak/datasets/creditcard.csv")
df.head()
df.Class.value_counts()
def show_confusion_matrix(confusion_matrix):
  hmap = sns.heatmap(confusion_matrix, annot=True, fmt="d",cmap="Blues")
  hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha="right")
  hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha="right")
  plt.xlabel("Prediction")
  plt.ylabel("Ground Truth")<jupyter_output><empty_output><jupyter_text>Isolation Forest<jupyter_code>train_df, test_df = train_test_split(df,test_size=0.33, random_state=SEED)
train_df, test_df = train_df.reset_index(drop=True), test_df.reset_index(drop=True)
print(train_df.shape, test_df.shape)
train_df['Class'] = np.where(train_df.Class == 0, 1, -1)
test_df['Class'] = np.where(test_df.Class == 0, 1, -1)
X_train = train_df.drop(['Class'], axis = 1)
y_train = train_df['Class']

X_test = test_df.drop(['Class'], axis = 1)
y_test = test_df['Class']
clfIF = IsolationForest(
    max_samples=0.25, 
    random_state=SEED, 
    contamination=0.05,#
    n_estimators=30, 
    n_jobs=-1
)
clfIF.fit(X_train, y_train)
y_pred_test = clfIF.predict(X_test)
print(classification_report(y_test, y_pred_test))
cm = confusion_matrix(y_test, y_pred_test)
df_cm = pd.DataFrame(
    cm,
    index = ["-1","1"], columns=["-1","1"]
)
show_confusion_matrix(df_cm)<jupyter_output><empty_output><jupyter_text>Isolation Forest With Subsampling<jupyter_code>train_df, test_df = train_test_split(df,test_size=0.33, random_state=SEED)
train_df, test_df = train_df.reset_index(drop=True), test_df.reset_index(drop=True)
print(train_df.shape, test_df.shape)
train_df['Class'] = np.where(train_df.Class == 0, 1, -1)
test_df['Class'] = np.where(test_df.Class == 0, 1, -1)
print(train_df[train_df["Class"] == -1].shape[0], train_df[train_df["Class"] == 1].shape[0])
n_samples = train_df[train_df["Class"] == 1].shape[0] // 35
anomalies = train_df[train_df["Class"] == -1].reset_index(drop=True)
subsample_normal = train_df[train_df["Class"] == 1].sample(n=n_samples, random_state=SEED).reset_index(drop=True)

sub_train_df = pd.concat([subsample_normal, anomalies], axis=0).sample(frac=1).reset_index(drop=True)
print(sub_train_df[sub_train_df["Class"] == -1].shape[0], sub_train_df[sub_train_df["Class"] == 1].shape[0])
X_train = sub_train_df.drop(['Class'], axis = 1)
y_train = sub_train_df['Class']

X_test = test_df.drop(['Class'], axis = 1)
y_test = test_df['Class']
clfIF = IsolationForest(
    max_samples = 0.25,
    contamination = 0.25,
    random_state=SEED, 
    n_estimators=30, 
    n_jobs=-1)
clfIF.fit(X_train, y_train)
y_pred_test = clfIF.predict(X_test)
print(classification_report(y_test, y_pred_test))
cm = confusion_matrix(y_test, y_pred_test)
df_cm = pd.DataFrame(
    cm,
    index = ["-1","1"], columns=["-1","1"]
)
show_confusion_matrix(df_cm)<jupyter_output><empty_output><jupyter_text>Local Outlier Factor<jupyter_code>train_df, test_df = train_test_split(df,test_size=0.33, random_state=SEED)
train_df, test_df = train_df.reset_index(drop=True), test_df.reset_index(drop=True)
print(train_df.shape, test_df.shape)
train_df['Class'] = np.where(train_df.Class == 0, 1, -1)
test_df['Class'] = np.where(test_df.Class == 0, 1, -1)
X_train = train_df.drop(['Class'], axis = 1)
y_train = train_df['Class']

X_test = test_df.drop(['Class'], axis = 1)
y_test = test_df['Class']
clfLOF = LocalOutlierFactor(
    n_neighbors=7, 
    metric='manhattan', 
    algorithm='auto', 
    contamination=0.05, 
    n_jobs=-1,
)
clfLOF.fit(X_train, y_train)
y_pred_test = clfLOF.fit_predict(X_test)
print(classification_report(y_test, y_pred_test))
cm = confusion_matrix(y_test, y_pred_test)
df_cm = pd.DataFrame(
    cm,
    index = ["-1","1"], columns=["-1","1"]
)
show_confusion_matrix(df_cm)<jupyter_output><empty_output><jupyter_text>Local Outlier Factor With Subsampling<jupyter_code>train_df, test_df = train_test_split(df,test_size=0.33, random_state=SEED)
train_df, test_df = train_df.reset_index(drop=True), test_df.reset_index(drop=True)
print(train_df.shape, test_df.shape)
train_df['Class'] = np.where(train_df.Class == 0, 1, -1)
test_df['Class'] = np.where(test_df.Class == 0, 1, -1)
n_samples = train_df[train_df["Class"] == 1].shape[0] // 200
anomalies = train_df[train_df["Class"] == -1].reset_index(drop=True)
subsample_normal = train_df[train_df["Class"] == 1].sample(n=n_samples, random_state=SEED).reset_index(drop=True)

sub_train_df = pd.concat([subsample_normal, anomalies], axis=0).sample(frac=1).reset_index(drop=True)
print(sub_train_df[sub_train_df["Class"] == -1].shape[0], sub_train_df[sub_train_df["Class"] == 1].shape[0])
X_train = sub_train_df.drop(['Class'], axis = 1)
y_train = sub_train_df['Class']

X_test = test_df.drop(['Class'], axis = 1)
y_test = test_df['Class']
clfLOF = LocalOutlierFactor(
    n_neighbors=7, 
    metric='l1', 
    algorithm='auto', 
    contamination=0.10,#len(y_train[y_train == -1])/len(y_train[y_train == 1]), 
    n_jobs=-1,
)
clfLOF.fit(X_train, y_train)
y_pred_test = clfLOF.fit_predict(X_test)
print(classification_report(y_test, y_pred_test))
cm = confusion_matrix(y_test, y_pred_test)
df_cm = pd.DataFrame(
    cm,
    index = ["-1","1"], columns=["-1","1"]
)
show_confusion_matrix(df_cm)<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>train_df, test_df = train_test_split(df,test_size=0.33, random_state=SEED)
train_df, test_df = train_df.reset_index(drop=True), test_df.reset_index(drop=True)
print(train_df.shape, test_df.shape)
def lets_make_some_xgb(X_train, X_test, y_train, y_test, useTrainCV=True, cv_folds=5,  early_stopping_rounds=50):
  clf = XGBClassifier(
      learning_rate=0.1, 
      n_estimators=140, 
      max_depth=5,
      min_child_weight=3, 
      gamma=0.2, 
      subsample=0.6,
      colsample_bytree=0.6,
      objective='binary:logistic', 
      nthread=4, 
      scale_pos_weight=20, 
      seed=27,
      tree_method = "gpu_hist"
  )

  if useTrainCV:
    print("Started cross-validation...")
    xgb_param = clf.get_xgb_params()
    xgtrain = xgb.DMatrix(X_train.values, label=y_train.values)
    cvresult = xgb.cv(
        xgb_param, 
        xgtrain, 
        num_boost_round=clf.get_params()['n_estimators'], 
        nfold=cv_folds,
        early_stopping_rounds=early_stopping_rounds
    )
    clf.set_params(n_estimators=cvresult.shape[0])
  print('Started Training...')
  clf.fit(X_train, y_train, eval_metric='auc')

  print("Started Predicting...")
  predictions = clf.predict(X_test)
  pred_proba = clf.predict_proba(X_test)[:, 1]

  print("\nResulting Metrics")
  print("Accuracy : %.4g" % accuracy_score(y_test, predictions))
  print("AUC : %f" % roc_auc_score(y_test, pred_proba))
  print("F1 Score : %f" % f1_score(y_test, predictions))
  print()
  print(f"{classification_report(y_test, predictions)}")
  print()
  cm = confusion_matrix(y_test, predictions)
  df_cm = pd.DataFrame(
      cm,
      index = ["0","1"], columns=["0","1"]
  )
  show_confusion_matrix(df_cm)
  print()

  feat_imp = clf.feature_importances_
  feat = X_train.columns.tolist()
  res_df = pd.DataFrame({'Features': feat, 'Importance': feat_imp}).sort_values(by='Importance', ascending=False)
  res_df.plot('Features', 'Importance', kind='bar', title='Feature Importances')
  plt.ylabel('Feature Importance Score')
  plt.show()
  print(res_df)
  print(res_df["Features"].tolist())
  return clf
X_train = train_df.drop(['Class'], axis = 1)
y_train = train_df['Class']

X_test = test_df.drop(['Class'], axis = 1)
y_test = test_df['Class']
clf = lets_make_some_xgb(X_train, X_test, y_train, y_test, useTrainCV=True)<jupyter_output>Started cross-validation...
Started Training...
Started Predicting...

Resulting Metrics
Accuracy : 0.9996
AUC : 0.985039
F1 Score : 0.875776

              precision    recall  f1-score   support

           0       1.00      1.00      1.00     93814
           1       0.95      0.82      0.88       173

    accuracy                           1.00     93987
   macro avg       0.97      0.91      0.94     93987
weighted avg       1.00      1.00      1.00     93987<jupyter_text>XGBoost With Subsampling<jupyter_code>train_df, test_df = train_test_split(df,test_size=0.33, random_state=SEED)
train_df, test_df = train_df.reset_index(drop=True), test_df.reset_index(drop=True)
print(train_df.shape, test_df.shape)
n_samples = train_df[train_df["Class"] == 0].shape[0] // 15
anomalies = train_df[train_df["Class"] == 1].reset_index(drop=True)
subsample_normal = train_df[train_df["Class"] == 0].sample(n=n_samples, random_state=SEED).reset_index(drop=True)

sub_train_df = pd.concat([subsample_normal, anomalies], axis=0).sample(frac=1).reset_index(drop=True)
print(sub_train_df[sub_train_df["Class"] == 0].shape[0], sub_train_df[sub_train_df["Class"] == 1].shape[0])
X_train = sub_train_df.drop(['Class'], axis = 1)
y_train = sub_train_df['Class']

X_test = test_df.drop(['Class'], axis = 1)
y_test = test_df['Class']
clf = lets_make_some_xgb(X_train, X_test, y_train, y_test, useTrainCV=True)<jupyter_output>Started cross-validation...
Started Training...
Started Predicting...

Resulting Metrics
Accuracy : 0.9986
AUC : 0.989380
F1 Score : 0.683721

              precision    recall  f1-score   support

           0       1.00      1.00      1.00     93814
           1       0.57      0.85      0.68       173

    accuracy                           1.00     93987
   macro avg       0.79      0.92      0.84     93987
weighted avg       1.00      1.00      1.00     93987<jupyter_text>Catboost<jupyter_code>train_df, test_df = train_test_split(df,test_size=0.33, random_state=SEED)
train_df, test_df = train_df.reset_index(drop=True), test_df.reset_index(drop=True)
print(train_df.shape, test_df.shape)
X_train = train_df.drop(['Class'], axis = 1)
y_train = train_df['Class']

X_test = test_df.drop(['Class'], axis = 1)
y_test = test_df['Class']
model = CatBoostClassifier(
    iterations=200,
    learning_rate=0.02,
    depth=12,
    eval_metric='AUC',
    random_seed = SEED,
    bagging_temperature = 0.2,
    od_type='Iter',
    od_wait=100,
    task_type="GPU",
    verbose=0
)

model.fit(X_train,y_train,eval_set=(X_test,y_test))
predictions = model.predict(X_test)
pred_proba = model.predict_proba(X_test)[:, 1]

print("\nResulting Metrics")
print("Accuracy : %.4g" % accuracy_score(y_test, predictions))
print("AUC : %f" % roc_auc_score(y_test, pred_proba))
print("F1 Score : %f" % f1_score(y_test, predictions))

print()
print(f"{classification_report(y_test, predictions)}")
print()
cm = confusion_matrix(y_test, predictions)
df_cm = pd.DataFrame(
    cm,
    index = ["0","1"], columns=["0","1"]
)
show_confusion_matrix(df_cm)<jupyter_output>Resulting Metrics
Accuracy : 0.9996
AUC : 0.980661
F1 Score : 0.876190

              precision    recall  f1-score   support

           0       1.00      1.00      1.00     93814
           1       0.97      0.80      0.88       173

    accuracy                           1.00     93987
   macro avg       0.99      0.90      0.94     93987
weighted avg       1.00      1.00      1.00     93987<jupyter_text>Catboost Hyperparameter Optimization<jupyter_code>train_df, test_df = train_test_split(df,test_size=0.33, random_state=SEED)
train_df, test_df = train_df.reset_index(drop=True), test_df.reset_index(drop=True)
print(train_df.shape, test_df.shape)
X_train = train_df.drop(['Class'], axis = 1)
y_train = train_df['Class']

X_test = test_df.drop(['Class'], axis = 1)
y_test = test_df['Class']
from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import uniform as sp_randFloat
from scipy.stats import randint as sp_randInt  

model = CatBoostClassifier(
    iterations=200,
    learning_rate=0.02,
    depth=12,
    eval_metric='AUC',
    random_seed = SEED,
    bagging_temperature = 0.2,
    od_type='Iter',
    od_wait=100,
    verbose=0
)

parameters = {'depth'         : sp_randInt(1, 20),
              'learning_rate' : sp_randFloat(),
              'iterations'    : sp_randInt(10, 200)
}
    
randm = RandomizedSearchCV(
    estimator=model, 
    param_distributions = 
    parameters, 
    cv = 2, 
    n_iter = 10, 
    n_jobs=-1
)
randm.fit(X_train, y_train, eval_set=(X_test,y_test))
print("Start Predicting")
predictions = randm.best_estimator_.predict(X_test)
pred_proba = randm.best_estimator_.predict_proba(X_test)[:, 1]

print("\nResulting Metrics")
print("Accuracy : %.4g" % accuracy_score(y_test, predictions))
print("AUC : %f" % roc_auc_score(y_test, pred_proba))
print("F1 Score : %f" % f1_score(y_test, predictions))

print()
print(f"{classification_report(y_test, predictions)}")
print()
cm = confusion_matrix(y_test, predictions)
df_cm = pd.DataFrame(
    cm,
    index = ["0","1"], columns=["0","1"]
)
show_confusion_matrix(df_cm)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14336677814938684, 'fraction_numerical': 0.037235228539576365, 'mean_word_length': 4.546578730420445, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '609380', 'license': 'MIT', 'n_tokens_mistral': 5514, 'n_tokens_neox': 5186, 'n_words': 833, 'path': 'Applied AI Study Group #5 - August 2021/Week 2/Anomaly Detection/Credit Card Fraud Detection.ipynb', 'repo_name': 'osmanbahtiyar/Applied-AI-Study-Group', 'size': '13455'}