<jupyter_start><jupyter_text>Import libs<jupyter_code>import numpy as np
from sklearn.model_selection import train_test_split
import pyspark.sql.functions as F
import pyod
import sys
sys.path.insert(0, 'models/')

import balance as bal
import cat_boost as cat
import isolation_forest
import parameters as param
import preprocessing as preprocess
import visualization as vis
import statistics as stats
import io_module as io

# This param when defined as True will execute the complete code, so slowly processing time
# because is require to execute all checks and print all not essential functions.
# When defined as False, a fast processing is applied with all core functionalities working
# well.
full_execution = False
verbose_mode = True

all_features = ['ProductId', 'ProductCategory', 'ChannelId', 'Value', 'PricingStrategy', 'Operation', 'PositiveAmount',
                'avg_ps_ChannelId', 'rt_avg_ps_ChannelId', 'avg_ps_ProductCategory', 'rt_avg_ps_ProductCategory',
                'avg_ps_ProductId', 'rt_avg_ps_ProductId']
columns_to_remove = ['CurrencyCode', 'CountryCode', 'BatchId', 'AccountId', 'SubscriptionId',
                     'CustomerId', 'TransactionStartTime', 'Amount']
categorical_features = ['ProductId', 'ProductCategory', 'ChannelId']
numerical_features = ['PositiveAmount', 'Operation', 'Value', 'PricingStrategy']
numerical_features_augmented = ['Value', 'PricingStrategy', 'Operation', 'PositiveAmount', 'avg_ps_ChannelId',
                                'rt_avg_ps_ChannelId', 'avg_ps_ProductCategory', 'rt_avg_ps_ProductCategory',
                                'avg_ps_ProductId', 'rt_avg_ps_ProductId']

label = 'FraudResult'
genuine_label = 'FraudResult==0'
fraud_label = 'FraudResult==1'
categorical_positions = [0, 1, 2, 4];<jupyter_output>Using TensorFlow backend.
/home/italo/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint8 = np.dtype([("qint8", np.int8, 1)])
/home/italo/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
/home/italo/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint16 = np.dtype([("qint16", np.int16, 1)])
/home/italo/anaconda3/lib/python3.7/site-packages/tensorflo[...]<jupyter_text>Read Data<jupyter_code># Read Fraud Detection Challenge data
train_data = io.read_spark_data_frame(param.get_file_name('training_data'))
train_data = preprocess.get_features_augmentation(train_data)
train_data = train_data.drop(*columns_to_remove)
x_data = train_data[numerical_features_augmented].toPandas()
y_data = train_data[[label]].toPandas()
from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=0.15, random_state=42)
from pyod.models.knn import KNN
from pyod.utils.data import evaluate_print

# train kNN detector
clf_name = 'KNN'
clf = KNN()
clf.fit(x_train);
# get the prediction labels and outlier scores of the training data
y_train_pred = clf.labels_  # binary labels (0: inliers, 1: outliers)
y_train_scores = clf.decision_scores_  # raw outlier scores

# get the prediction on the test data
y_test_pred = clf.predict(x_test)  # outlier labels (0 or 1)
y_test_scores = clf.decision_function(x_test)  # outlier scores
# evaluate and print the results
print("\nOn Training Data:")
evaluate_print(clf_name, y_train, y_train_scores)
print("\nOn Test Data:")
evaluate_print(clf_name, y_test, y_test_scores)<jupyter_output><empty_output><jupyter_text>With combination<jupyter_code>from pyod.models.combination import aom, moa, average, maximization
from pyod.models.abod import ABOD
from pyod.models.cblof import CBLOF
from pyod.models.feature_bagging import FeatureBagging
from pyod.models.hbos import HBOS
from pyod.models.iforest import IForest
from pyod.models.knn import KNN
from pyod.models.lof import LOF
from pyod.models.loci import LOCI
from pyod.models.mcd import MCD
from pyod.models.ocsvm import OCSVM
from pyod.models.pca import PCA
from pyod.models.sos import SOS
from pyod.models.lscp import LSCP
from pyod.models.cof import COF
from pyod.models.sod import SOD

from pyod.models.xgbod import XGBOD
clf = XGBOD()
#clf = LSCP()
clf.fit(x_train, np.array(y_train));<jupyter_output><empty_output><jupyter_text>Testing data<jupyter_code># Read Fraud Detection Challenge data
test_data = io.read_spark_data_frame(param.get_file_name('testing_data'))
test_data = preprocess.get_features_augmentation(test_data)
test_data = train_data.drop(*columns_to_remove)
x_data = test_data[numerical_features_augmented].toPandas()
y_test_pred = clf.predict(x_data)
io.save_predictions_xente('../data/predictions_000.txt', transactions_list, y_test_pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10607210626185959, 'fraction_numerical': 0.012713472485768502, 'mean_word_length': 5.845454545454546, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '312840', 'license': 'MIT', 'n_tokens_mistral': 1853, 'n_tokens_neox': 1776, 'n_words': 448, 'path': 'jupyter-notebook/PyOD analysis.ipynb', 'repo_name': 'fernandojunior/financial-fraud-detection', 'size': '5270'}