<jupyter_start><jupyter_text>Template - Author: <NAME> [\[e-mail\]](mailto:'Israel%20Oliveira%20')<jupyter_code>!pip3 install -U random-forest-mc
%load_ext watermark
import warnings
import pandas as pd
import numpy as np
from random_forest_mc.model import RandomForestMC
from random_forest_mc.utils import load_file_json, dump_file_json, LoadDicts
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score, roc_auc_score, recall_score
from collections import Counter
import hashlib
from joblib import dump

warnings.simplefilter(action='ignore', category=FutureWarning)
pd.options.mode.chained_assignment = None
from tqdm import tqdm

# from glob import glob

# import matplotlib.pyplot as plt
# %matplotlib inline
# from matplotlib import rcParams
# from cycler import cycler

# rcParams['figure.figsize'] = 12, 8 # 18, 5
# rcParams['axes.spines.top'] = False
# rcParams['axes.spines.right'] = False
# rcParams['axes.grid'] = True
# rcParams['axes.prop_cycle'] = cycler(color=['#365977'])
# rcParams['lines.linewidth'] = 2.5

# import seaborn as sns
# sns.set_theme()

# pd.set_option("max_columns", None)
# pd.set_option("max_rows", None)
# pd.set_option('display.max_colwidth', None)

from IPython.display import Markdown, display
def md(arg):
    display(Markdown(arg))

# from pandas_profiling import ProfileReport
# #report = ProfileReport(#DataFrame here#, minimal=True)
# #report.to

# import pyarrow.parquet as pq
# #df = pq.ParquetDataset(path_to_folder_with_parquets, filesystem=None).read_pandas().to_pandas()

# import json
# def open_file_json(path,mode='r',var=None):
#     if mode == 'w':
#         with open(path,'w') as f:
#             json.dump(var, f)
#     if mode == 'r':
#         with open(path,'r') as f:
#             return json.load(f)

# import functools
# import operator
# def flat(a):
#     return functools.reduce(operator.iconcat, a, [])

# import json
# from glob import glob
# from typing import NewType


# DictsPathType = NewType("DictsPath", str)


# def open_file_json(path):
#     with open(path, "r") as f:
#         return json.load(f)

# class LoadDicts:
#     def __init__(self, dict_path: DictsPathType = "./data"):
#         Dicts_glob = glob(f"{dict_path}/*.json")
#         self.List = []
#         self.Dict = {}
#         for path_json in Dicts_glob:
#             name = path_json.split("/")[-1].replace(".json", "")
#             self.List.append(name)
#             self.Dict[name] = open_file_json(path_json)
#             setattr(self, name, self.Dict[name])

# Run this cell before close.
%watermark -d --iversion -b -r -g -m -v
!cat /proc/cpuinfo |grep 'model name'|head -n 1 |sed -e 's/model\ name/CPU/'
!free -h |cut -d'i' -f1  |grep -v total
dicts = LoadDicts("../tests/")
dataset_dict = dicts.datasets_metadata
dataset_dict.keys()
def dfany2dfnumeric(dfA, dfB):
    for col in dfA.columns:
        mapper = {val: i for i,val in enumerate(dfA[col].unique().tolist()+dfB[col].unique().tolist())}
        dfA[col] = dfA[col].apply(lambda x: mapper[x])
        dfB[col] = dfB[col].apply(lambda x: mapper[x])
    return dfA, dfB<jupyter_output><empty_output><jupyter_text>Tinanic<jupyter_code>ds_name = "titanic"
params = dataset_dict[ds_name]
dataset = (
    pd.read_csv(params["csv_path"].replace('~','/work'))[params["ds_cols"] + [params["target_col"]]]
    .dropna()
    .reset_index(drop=True)
)
dataset["Age"] = dataset["Age"].astype(np.uint8)
dataset["SibSp"] = dataset["SibSp"].astype(np.uint8)
dataset["Pclass"] = dataset["Pclass"].astype(str)
dataset["Fare"] = dataset["Fare"].astype(np.uint32)
target_col = params['target_col']
dataset = dataset[params['ds_cols']+[target_col]].dropna().reset_index(drop=True)
dataset[target_col] = dataset[target_col].astype(str)

test_size = 0.5
max_test_size = 10000
max_train_size = 10000

split_seeds = [43, 47, 53, 59]
#def fullCycleAnalysis(dataset,params,test_size_frac, max_test_size, max_train_size, split_seeds, )
target_col = params['target_col']
ds_cols = params["ds_cols"]

Results = []
# Each seed is a experiment.
for seed in split_seeds:
    
    df_train_list = []
    df_test_list = []
    df_classes = [dataset.query(f'{target_col} == "{target_val}"').reset_index(drop=True).copy() for target_val in dataset[target_col].unique()]
    while df_classes:
        df = df_classes.pop(0)
        df_train, df_test = train_test_split(df, test_size=test_size, random_state=seed)
        if df_train.shape[0] > max_train_size:
            df_train = df_train.sample(n = max_train_size, random_state=seed)
        if df_test.shape[0] > max_test_size:
            df_test = df_test.sample(n = max_test_size, random_state=seed)
        df_train_list.append(df_train.copy())
        df_test_list.append(df_test.copy())
        del df
    dataset_train = pd.concat(df_train_list)
    dataset_train = dataset_train.sample(dataset_train.shape[0]).reset_index(drop=True).copy()
    del df_train_list
    dataset_test = pd.concat(df_test_list)
    dataset_test = dataset_test.sample(dataset_test.shape[0]).reset_index(drop=True).copy()
    del df_test_list

    y_test = dataset_test[target_col].tolist()
    NT = dataset_train[target_col].value_counts().sort_values().min() 
    # Training step
    cls = RandomForestMC(
        n_trees=64, target_col=target_col, max_discard_trees=256,
        th_start=0.9999, batch_train_pclass = NT // 2, batch_val_pclass = NT // 2
    )
    cls.process_dataset(dataset_train)
    cls.fitParallel(max_workers=16, thread_parallel_method=False)
    modeldict = cls.model2dict()
    dump_file_json(f'/work/data/RandomForestMC_{ds_name}_{seed}.json', modeldict)
    y_pred = cls.testForest(dataset_test)
    F1_M = f1_score(y_test, y_pred, average='macro')
    AUC_ROC_M = roc_auc_score(y_test, y_pred, average='macro')
    RECALL = recall_score(y_test, y_pred, pos_label='1')
    TP_1 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '1'])/y_test.count('1')
    TP_0 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '0'])/y_test.count('0')
    
    Results.append((seed, 'RFMC', F1_M, AUC_ROC_M, RECALL, TP_1, TP_0))
    
    dataset_train_num, dataset_test_num = dfany2dfnumeric(dataset_train[ds_cols], dataset_test[ds_cols])
    # scaler = StandardScaler()
    # scaler.fit(dataset_train_num[ds_cols].to_numpy())  
    # X_train = scaler.transform(dataset_train_num[ds_cols].to_numpy())  
    # X_test = scaler.transform(dataset_test_num[ds_cols].to_numpy())  
    
    cls = RandomForestClassifier(n_estimators=400, n_jobs=-1)
    cls.fit(dataset_train_num[ds_cols].to_numpy(), dataset_train[target_col].to_numpy())
    dump(cls, f'/work/data/RandomForestClassifier_{ds_name}_{seed}.joblib')
    y_pred = cls.predict(dataset_test_num[ds_cols].to_numpy()).tolist()
    F1_M = f1_score(y_test, y_pred, average='macro')
    AUC_ROC_M = roc_auc_score(y_test, y_pred, average='macro')
    RECALL = recall_score(y_test, y_pred, pos_label='1')
    TP_1 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '1'])/y_test.count('1')
    TP_0 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '0'])/y_test.count('0')
    
    Results.append((seed, 'RF', F1_M, AUC_ROC_M, RECALL, TP_1, TP_0))   
    
    cls = MLPClassifier(hidden_layer_sizes=(200,), max_iter=10000)
    cls.fit(dataset_train_num[ds_cols].to_numpy(), dataset_train[target_col].to_numpy())
    dump(cls, f'/work/data/MLPClassifier_{ds_name}_{seed}.joblib')
    y_pred = cls.predict(dataset_test_num[ds_cols].to_numpy()).tolist()
    F1_M = f1_score(y_test, y_pred, average='macro')
    AUC_ROC_M = roc_auc_score(y_test, y_pred, average='macro')
    RECALL = recall_score(y_test, y_pred, pos_label='1')
    TP_1 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '1'])/y_test.count('1')
    TP_0 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '0'])/y_test.count('0')
    
    Results.append((seed, 'MLPC', F1_M, AUC_ROC_M, RECALL, TP_1, TP_0))
    
    del dataset_train, dataset_train_num, dataset_test_num
    
columns=['seed', 'MODEL', 'F1_M', 'AUC_ROC_M', 'RECALL', 'TP_1', 'TP_0']
df_results = pd.DataFrame(Results, columns=columns)
for model in df_results.MODEL.unique():
    md(f'## {model}')
    display(df_results[columns[1:]].query(f'MODEL == "{model}"').describe())
    
df_results.to_csv(f'/work/data/comparison_experiments_{ds_name}.csv', index=False)<jupyter_output><empty_output><jupyter_text>Credit Card Fraude Detection<jupyter_code>ds_name = "creditcard_trans_int"
params = dataset_dict[ds_name]
dataset = (
    pd.read_csv(params["csv_path"].replace('~','/work'))[params["ds_cols"] + [params["target_col"]]]
    .dropna()
    .reset_index(drop=True)
)
target_col = params['target_col']
dataset = dataset[params['ds_cols']+[target_col]].dropna().reset_index(drop=True)
dataset[target_col] = dataset[target_col].astype(str)

test_size = 0.5
max_test_size = 10000
max_train_size = 2000

split_seeds = [43, 47, 53, 59]
#def fullCycleAnalysis(dataset,params,test_size_frac, max_test_size, max_train_size, split_seeds, )
target_col = params['target_col']
ds_cols = params["ds_cols"]

Results = []
# Each seed is a experiment.
for seed in split_seeds:
    
    df_train_list = []
    df_test_list = []
    df_classes = [dataset.query(f'{target_col} == "{target_val}"').reset_index(drop=True).copy() for target_val in dataset[target_col].unique()]
    while df_classes:
        df = df_classes.pop(0)
        df_train, df_test = train_test_split(df, test_size=test_size, random_state=seed)
        if df_train.shape[0] > max_train_size:
            df_train = df_train.sample(n = max_train_size, random_state=seed)
        if df_test.shape[0] > max_test_size:
            df_test = df_test.sample(n = max_test_size, random_state=seed)
        df_train_list.append(df_train.copy())
        df_test_list.append(df_test.copy())
        del df
    dataset_train = pd.concat(df_train_list)
    dataset_train = dataset_train.sample(dataset_train.shape[0]).reset_index(drop=True).copy()
    del df_train_list
    dataset_test = pd.concat(df_test_list)
    dataset_test = dataset_test.sample(dataset_test.shape[0]).reset_index(drop=True).copy()
    del df_test_list

    y_test = dataset_test[target_col].astype(str).tolist()
    NT = dataset_train[target_col].value_counts().sort_values().min() 
    # Training step
    cls = RandomForestMC(
        n_trees=64, target_col=target_col, max_discard_trees=256,
        th_start=0.9999, batch_train_pclass = NT // 2, batch_val_pclass = NT // 2
    )
    cls.process_dataset(dataset_train)
    cls.fitParallel(max_workers=16, thread_parallel_method=False)
    modeldict = cls.model2dict()
    dump_file_json(f'/work/data/RandomForestMC_{ds_name}_{seed}.json', modeldict)
    y_pred = cls.testForest(dataset_test)
    F1_M = f1_score(y_test, y_pred, average='macro')
    AUC_ROC_M = roc_auc_score(y_test, y_pred, average='macro')
    RECALL = recall_score(y_test, y_pred, pos_label='1')
    TP_1 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '1'])/y_test.count('1')
    TP_0 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '0'])/y_test.count('0')
    
    Results.append((seed, 'RFMC', F1_M, AUC_ROC_M, RECALL, TP_1, TP_0))
    
    dataset_train_num, dataset_test_num = dataset_train[ds_cols], dataset_test[ds_cols]
    # scaler = StandardScaler()
    # scaler.fit(dataset_train_num[ds_cols].to_numpy())  
    # X_train = scaler.transform(dataset_train_num[ds_cols].to_numpy())  
    # X_test = scaler.transform(dataset_test_num[ds_cols].to_numpy())  
    
    cls = RandomForestClassifier(n_estimators=400, n_jobs=-1)
    cls.fit(dataset_train_num[ds_cols].to_numpy(), dataset_train[target_col].to_numpy())
    dump(cls, f'/work/data/RandomForestClassifier_{ds_name}_{seed}.joblib')
    
    y_pred = cls.predict(dataset_test_num[ds_cols].to_numpy()).tolist()
    F1_M = f1_score(y_test, y_pred, average='macro')
    AUC_ROC_M = roc_auc_score(y_test, y_pred, average='macro')
    RECALL = recall_score(y_test, y_pred, pos_label='1')
    TP_1 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '1'])/y_test.count('1')
    TP_0 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '0'])/y_test.count('0')
    
    Results.append((seed, 'RF', F1_M, AUC_ROC_M, RECALL, TP_1, TP_0))   
    
    cls = MLPClassifier(hidden_layer_sizes=(200,), max_iter=10000)
    cls.fit(dataset_train_num[ds_cols].to_numpy(), dataset_train[target_col].to_numpy())
    dump(cls, f'/work/data/MLPClassifier_{ds_name}_{seed}.joblib')
    y_pred = cls.predict(dataset_test_num[ds_cols].to_numpy()).tolist()
    F1_M = f1_score(y_test, y_pred, average='macro')
    AUC_ROC_M = roc_auc_score(y_test, y_pred, average='macro')
    RECALL = recall_score(y_test, y_pred, pos_label='1')
    TP_1 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '1'])/y_test.count('1')
    TP_0 = sum([ p == t for p,t in zip(y_pred,y_test) if t == '0'])/y_test.count('0')
    
    Results.append((seed, 'MLPC', F1_M, AUC_ROC_M, RECALL, TP_1, TP_0))
    
    del dataset_train, dataset_train_num, dataset_test_num
    
columns=['seed', 'MODEL', 'F1_M', 'AUC_ROC_M', 'RECALL', 'TP_1', 'TP_0']
df_results = pd.DataFrame(Results, columns=columns)
for model in df_results.MODEL.unique():
    md(f'## {model}')
    display(df_results[columns[1:]].query(f'MODEL == "{model}"').describe())
    
df_results.to_csv(f'/work/data/comparison_experiments_{ds_name}.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14909359970403255, 'fraction_numerical': 0.01568627450980392, 'mean_word_length': 4.569015245158632, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '455975', 'license': 'MIT', 'n_tokens_mistral': 5299, 'n_tokens_neox': 5138, 'n_words': 876, 'path': 'utils/Comparison_Experiments.ipynb', 'repo_name': 'ysraell/random-forest', 'size': '13524'}