<jupyter_start><jupyter_text>**Imports and configurations**<jupyter_code>%reload_ext autoreload
%autoreload 2
import os
import time

import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import plotly.graph_objs as go

from IPython.display import IFrame
from plotly.offline import init_notebook_mode
init_notebook_mode(connected=True)

from plot_utils import *
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.ensemble import RandomForestClassifier

from sklearn.utils import shuffle
from sklearn.model_selection import RepeatedStratifiedKFold, cross_val_score
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
folder_path = 'C:/Users/yaass/OneDrive/Bureau/Parser'<jupyter_output><empty_output><jupyter_text>**Models evaluation**<jupyter_code>def preview_data(folder_path, file_name, index_value = 'md5'):
    df = pd.read_csv(os.path.join(folder_path, file_name), index_col=index_value)
    print(f'Nb Observations: {df.shape[0]}')
    print(f'Nb Columns: {df.shape[1]}')
    return df.head(3)


def create_X_y(folder_path, file_name, index_value = 'md5'):
    file_path = os.path.join(folder_path, file_name)
    df = pd.read_csv(file_path, index_col = index_value)
    X = df.drop('label', axis=1)
    y = df['label']
    return shuffle(X, y)


def create_regular_net():
    model = Sequential()
    model.add(Dense(units=10, kernel_initializer = 'uniform', activation = 'relu', name='dense_layer1'))
    model.add(Dense(units=10, kernel_initializer = 'uniform', activation = 'relu', name='dense_layer2'))
    model.add(Dense(1, activation = 'sigmoid', name = 'dense_output'))   #sigmoid for binary 
    model.compile(loss='binary_crossentropy', optimizer= 'adam', metrics = ['accuracy'])
    return model


def wrap_regular_net():
    model = KerasClassifier(build_fn=create_regular_net, epochs=50, batch_size=64, verbose=0)
    return model


def create_models():
    models = dict()
    models['LogisticRegression'] = LogisticRegression()
    models['KNN'] = KNeighborsClassifier()
    models['Decision tree'] = DecisionTreeClassifier()
    models['Random Forest'] = RandomForestClassifier()
    #models['Stochastic Gradient Descent'] = SGDClassifier()
    models['SVM'] = SVC()
    models['RegularNets'] = wrap_regular_net()
    models['LDA'] = LDA()
    models['Gaussian Naive Bayes'] = GaussianNB()
    return models


def evaluate_model(model, X, y):
    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
    scores = cross_val_score(model, X, y, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')
    return scores


def get_evaluation_results(folder_path, file_name):  
    
    #generate features and target
    X, y = create_X_y(folder_path, file_name)
    
    #generate models
    models = create_models()
    
    #evaluate models and store results
    results, names = list(), list()
    for name, model in models.items():
        scores = evaluate_model(model, X, y)
        results.append(scores)
        names.append(name) 
        
    return names, results


def print_evaluation_results(results, names):  
    for name, scores in zip(names, results):
        print(f"{name:30} \t: {np.mean(scores)*100:.3f}% ( {np.std(scores)*100:.3f}% )")<jupyter_output><empty_output><jupyter_text>1. apistats (one-hot encoded)<jupyter_code>file_name = 'onehot_encoded_apistats_dataset.csv'

preview_data(folder_path, file_name)
start = time.time()

names, results = get_evaluation_results(folder_path, file_name)

print(f'Cross-validation {len(names)} models required a duration of {time.time()-start:.2f} seconds')
print_evaluation_results(results, names)
fig = plot_evaluation_boxplots(results, names, title = 'Models Performance On One-Hot Encoded "apistats" Data', y_axis = 'Accuracy')

figure_path = 'figures/evaluation_onehot_apistats_data.html'

save_figures_to_html(figure_path, [fig])

IFrame(figure_path, width=900, height=600)<jupyter_output><empty_output><jupyter_text>1. apistats (count encoded)<jupyter_code>file_name = 'count_encoded_apistats_dataset.csv'

preview_data(folder_path, file_name)
start = time.time()

names, results = get_evaluation_results(folder_path, file_name)

print(f'Cross-validation on {len(names)} models required a duration of {time.time()-start:.2f} seconds')
print_evaluation_results(results, names)
fig = plot_evaluation_boxplots(results, names, title = 'Models Performance On Count Encoded "apistats" Data', y_axis = 'Accuracy')

figure_path = 'figures/evaluation_count_apistats_data.html'

save_figures_to_html(figure_path, [fig])

IFrame(figure_path, width=900, height=600)<jupyter_output><empty_output><jupyter_text>3. dll_loaded (one-hot encoded)<jupyter_code>file_name = 'onehot_encoded_dll_dataset.csv'

preview_data(folder_path, file_name)
start = time.time()

names, results = get_evaluation_results(folder_path, file_name)

print(f'Cross-validation {len(names)} models required a duration of {time.time()-start:.2f} seconds')
print_evaluation_results(results, names)
fig = plot_evaluation_boxplots(results, names, title = 'Models Performance On One-Hot Encoded "dll_loaded" Data', y_axis = 'Accuracy')

figure_path = 'figures/evaluation_onehot_dll_data.html'

save_figures_to_html(figure_path, [fig])

IFrame(figure_path, width=900, height=600)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11248236953455572, 'fraction_numerical': 0.010225669957686883, 'mean_word_length': 6.126884422110553, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '70559', 'license': 'CC0-1.0', 'n_tokens_mistral': 1988, 'n_tokens_neox': 1899, 'n_words': 401, 'path': 'model-selection/initial-model-selection.ipynb', 'repo_name': 'milxss/ransomware-preencryption-detector', 'size': '5672'}