<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import sys
import pandas as pd
import numpy as np
import multiprocessing as mp
import gc

from sklearn import preprocessing
from sklearn.decomposition import PCA
from sklearn import random_projection
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import fbeta_score, roc_curve, auc
from sklearn import svm
from sklearn.ensemble import IsolationForest

from itertools import product
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import pickle
import json

import time

num_partitions = 10 #number of partitions to split dataframe
num_cores = mp.cpu_count() #number of cores on your machine

pd.options.display.max_columns = 999
p = mp.Pool(mp.cpu_count()) # Data parallelism Object
sys.path.insert(0, '../../scripts/modeling_toolbox/')
# load the autoreload extension
%load_ext autoreload
# Set extension to reload modules every time before executing code
%autoreload 2

from metric_processor import MetricProcessor
import evaluation

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Auxiliary functions and lists<jupyter_code>def parallelize_dataframe(df, func):
    df_split = np.array_split(df, num_partitions)
    pool = mp.Pool(num_cores)
    df = pd.concat(pool.map(func, df_split))
    pool.close()
    pool.join()
    del df_split
    return df
def convert_to_numpy(df):
    
    for column in df.columns:
        df[column] = df[column].apply(lambda x: np.fromstring(x.replace('[', '').replace(']', ''),
                          dtype=np.float,
                          sep=' '))
        
    return df
def compute_mean(df):
    
    for column in df.columns:
        df[column] = df[column].apply(lambda x: x.mean())
        
    return df
def compute_mean_samples(df):
    global samples_number
    
    for column in df.columns:
        df[column] = df[column].apply(lambda x: np.mean(list(np.random.choice(x, samples_number))))
    
    return df
def split_test_and_train(df, train_prop=0.8):

    df_1 = df[df['attack_ID'] < 10]
    df_0 = df[df['attack_ID'] >= 10]

    num_train = int(df_1.shape[0]*train_prop)
    df_train = df_1[0:num_train]
    df_test = df_1[num_train:]
    df_attacks = df_0

    df_train = df_train.sample(frac=1)
    df_test = df_test.sample(frac=1)
    df_attacks = df_attacks.sample(frac=1)

    x_train = df_train.drop(['attack_ID'], axis=1)
    x_train = np.asarray(x_train)

    x_test = df_test.drop(['attack_ID'], axis=1)
    x_test = np.asarray(x_test)

    x_attacks = df_attacks.drop(['attack_ID'], axis=1)
    x_attacks = np.asarray(x_attacks)

    return (x_train, x_test, x_attacks), (df_train, df_test, df_attacks)
downscale_features = ['temporal_psnr',
                      'temporal_ssim',
                      'temporal_cross_correlation'
                     ]

upscale_features = ['temporal_difference',
                    'temporal_dct',
                    'temporal_canny',
                    'temporal_gaussian_mse',
                    'temporal_gaussian_difference',
                    'temporal_histogram_distance',
                    'temporal_entropy',
                    'temporal_lbp',
                    'temporal_texture',
                    'temporal_match',
                   ]

features = ['dimension',
            'size',
            'temporal_dct-mean', 
            'temporal_gaussian_mse-mean', 
            'temporal_gaussian_difference-mean',
            'temporal_threshold_gaussian_difference-mean',
           ]<jupyter_output><empty_output><jupyter_text>Data Preparation<jupyter_code>path = '../../machine_learning/cloud_functions/data-large.csv'
reduced = False

data = pd.read_csv(path)
if reduced:
    data = data[:reduced]

df = pd.DataFrame(data)

del data

print('ORIGINAL DATASET:')
display(df.head())

df['attack'] = df['attack'].apply(lambda x: MetricProcessor.set_attack_name(x))
df['attack_ID'] = df['attack'].apply(lambda x: MetricProcessor.set_attack_id(x))
df['size_dimension_ratio'] = df['size'] / df['dimension']
print(df.shape)
display(df.head())
print('Sampling dataframe')
time_series_df = df[[column for column in df.columns if 'series' in column]]
display(time_series_df.head())
samples_number = 60

start_time = time.time()

time_series_df = parallelize_dataframe(time_series_df, convert_to_numpy)

elapsed_time = time.time() - start_time
print('Conversion time:', elapsed_time)

start_time = time.time()
%reset -f out
display(time_series_df.head())
mean_values_df = parallelize_dataframe(time_series_df, compute_mean)
elapsed_time = time.time() - start_time
print('Mean computation time:', elapsed_time)
mean_values_df['dimension'] = df['dimension']
mean_values_df['size_dimension_ratio'] = df['size_dimension_ratio']
mean_values_df['attack_ID'] = df['attack_ID']
for column in time_series_df.columns:
    
    for label in downscale_features:
        if label in column:
            print('Upscaling', label)
            mean_values_df[column] = mean_values_df[column] / mean_values_df['dimension']
    for label in upscale_features:
        if label in column:
            print('Downscaling', label)
            mean_values_df[column] = mean_values_df[column] * mean_values_df['dimension']
display(mean_values_df)

(X_train, X_test, X_attacks), (df_train, df_test, df_attacks) = split_test_and_train(mean_values_df)

print('Shape of train: {}'.format(X_train.shape))
print('Shape of test: {}'.format(X_test.shape))
print('Shape of attacks: {}'.format(X_attacks.shape))

# Scaling the data
ss = StandardScaler()
x_train = ss.fit_transform(X_train)
x_test = ss.transform(X_test)
x_attacks = ss.transform(X_attacks)

# One Class SVM

# Dataframe to store results
svm_results = pd.DataFrame(columns=['gamma', 'nu', 'n_components', 'TPR_test',
                                    'TNR', 'model', 'auc', 'f_beta', 'projection'])

# Train the models
svm_results = evaluation.one_class_svm(x_train, x_test, x_attacks, svm_results)
display(svm_results.sort_values('f_beta', ascending=False).head())

# Save the best model
best_svm = svm_results.sort_values('f_beta', ascending=False).iloc[0]
projection = best_svm['projection']

reduction = None
if projection == 'PCA':
    reduction = PCA(n_components=best_svm['n_components'])
else:
    print('Unknown projection type')
    X_reduced = x_train
    attack_reduced = x_attacks
    test_reduced = x_test
    
if reduction:    
    X_reduced = reduction.fit_transform(x_train)
    attack_reduced = reduction.transform(x_attacks)
    test_reduced = reduction.transform(x_test)
    pickle.dump(reduction, open('../output/models/reduction_OCSVM.pickle.dat', 'wb'))


OCSVM = svm.OneClassSVM(kernel='rbf',gamma=best_svm['gamma'], nu=best_svm['nu'], cache_size=5000)

OCSVM.fit(X_reduced)

    
samples_series = [60]

sample_df = pd.DataFrame(columns=['#samples', 'f20', 'tnr', 'tpr_train', 'tpr_test'])
for n in samples_series:
    print('Number of samples:', n)
    samples_number = n
    for i in range(1000):
        
        start_time = time.time()
        
        mean_values_df = parallelize_dataframe(time_series_df, compute_mean_samples)
        elapsed_time = time.time() - start_time
        mean_values_df['dimension'] = df['dimension']
        mean_values_df['size_dimension_ratio'] = df['size_dimension_ratio']
        mean_values_df['attack_ID'] = df['attack_ID']
        for column in time_series_df.columns:

            for label in downscale_features:
                if label in column:
                    mean_values_df[column] = mean_values_df[column] / mean_values_df['dimension']
            for label in upscale_features:
                if label in column:
                    mean_values_df[column] = mean_values_df[column] * mean_values_df['dimension']

        
        (X_train, X_test, X_attacks), (df_train, df_test, df_attacks) = split_test_and_train(mean_values_df)

        # Scaling the data
        ss = StandardScaler()
        x_train = ss.fit_transform(X_train)
        x_test = ss.fit_transform(X_test)
        x_attacks = ss.transform(X_attacks)

        fb, area, tnr, tpr_train, tpr_test = evaluation.unsupervised_evaluation(OCSVM, x_train,
                                                                             x_test, x_attacks)
        sample_df = sample_df.append({'#samples': n,
                                      'f20': fb,
                                      'tnr': tnr,
                                      'tpr_train': tpr_train,
                                      'tpr_test': tpr_test},
                                     ignore_index=True)
        display(sample_df)
        del mean_values_df, X_train, X_test, X_attacks, df_train, df_test, df_attacks
        gc.collect()
        elapsed_time = time.time() - start_time
        print('Computation time:', elapsed_time)
    sample_df.to_csv('Samples-{}.csv'.format(n))
    display(sample_df)<jupyter_output><empty_output><jupyter_text>Compute PDFs and make t-test to extract confidence intervals<jupyter_code>data = pd.read_csv('./samples/Samples-40.csv')
sample_df = pd.DataFrame(data)
data = pd.read_csv('./samples/Samples-55.csv')
sample_df = pd.concat([sample_df, pd.DataFrame(data)])
sample_df = sample_df.drop('Unnamed: 0', axis=1)
print('ORIGINAL DATASET:')
print(sample_df.shape)
display(sample_df.head())
display(sample_df.groupby('#samples').count())
display(sample_df.describe())<jupyter_output><empty_output><jupyter_text>PDF plots:<jupyter_code>import plotly.tools
import plotly.plotly as py
import plotly.graph_objs as go
import plotly.offline as offline
from plotly import tools

offline.init_notebook_mode()

row = 1
col = 1
rows = 3
cols = 4
fig = tools.make_subplots(
                          rows=rows,
                          cols=cols,
                          shared_yaxes=True)

samples_series = [5, 10, 15, 20 , 25, 30, 35, 40, 45, 50, 55]
variable = 'f20'

for sample_number in samples_series:
    if row == rows + 1:
        row = 1
        col += 1
    samples_x_frames_df = sample_df[sample_df['#samples']==sample_number]
    X = samples_x_frames_df[variable].values

    fig.add_trace(go.Histogram(x=X,
                               name=sample_number,
                               xbins=dict( # bins used for histogram
                                          start=0.94,
                                          end=0.98,
                                          size=0.0001
                                         ),
                                opacity=0.75
                               ),
                  row=row,
                  col=col
                )
    row +=1
 
offline.iplot(fig)<jupyter_output><empty_output><jupyter_text>Statistical Normality TestsAs it is nicely explained [here](https://machinelearningmastery.com/a-gentle-introduction-to-normality-tests-in-python/)**Interpretation of a Test**Before you can apply the statistical tests, you must know how to interpret the results.Each test will return at least two things:   - Statistic: A quantity calculated by the test that can be interpreted in the context of the test via comparing it to critical values from the distribution of the test statistic.   - p-value: Used to interpret the test, in this case whether the sample was drawn from a Gaussian distribution.Each test calculates a test-specific statistic. This statistic can aid in the interpretation of the result, although it may require a deeper proficiency with statistics and a deeper knowledge of the specific statistical test. Instead, the p-value can be used to quickly and accurately interpret the statistic in practical applications.The tests assume that that the sample was drawn from a Gaussian distribution. Technically this is called the null hypothesis, or H0. A threshold level is chosen called alpha, typically 5% (or 0.05), that is used to interpret the p-value.In the SciPy implementation of these tests, you can interpret the p value as follows.   - p <= alpha: reject H0, not normal.   - p > alpha: fail to reject H0, normal.This means that, in general, we are seeking results with a larger p-value to confirm that our sample was likely drawn from a Gaussian distribution.A result above 5% does not mean that the null hypothesis is true. It means that it is very likely true given available evidence. The p-value is not the probability of the data fitting a Gaussian distribution; it can be thought of as a value that helps us interpret the statistical test.**Shapiro-Wilk Test**The Shapiro-Wilk test evaluates a data sample and quantifies how likely it is that the data was drawn from a Gaussian distribution, named for <NAME> and <NAME>.In practice, the Shapiro-Wilk test is believed to be a reliable test of normality, although there is some suggestion that the test may be suitable for smaller samples of data, e.g. thousands of observations or fewer**D’Agostino’s K^2 Test**The D’Agostino’s K^2 test calculates summary statistics from the data, namely kurtosis and skewness, to determine if the data distribution departs from the normal distribution, named for <NAME>.   - Skew is a quantification of how much a distribution is pushed left or right, a measure of asymmetry in the distribution.   - Kurtosis quantifies how much of the distribution is in the tail. It is a simple and commonly used statistical test for normality.The D’Agostino’s K^2 test is available via the normaltest() SciPy function and returns the test statistic and the p-value.<jupyter_code># Compute the data normality
from scipy.stats import shapiro
from scipy.stats import normaltest

test_methods = ['Shapiro', 'DAgostino']

normality_stats_df = pd.DataFrame(columns=test_methods)
normality_p_df = pd.DataFrame(columns=test_methods)

for sample_number in samples_series:

    samples_x_frames_df = sample_df[sample_df['#samples']==sample_number]
    
    stat_shapiro, p_shapiro = shapiro(samples_x_frames_df[variable])
    stat_dagostino, p_dagostino = normaltest(samples_x_frames_df[variable])
    
    normality_p_df = normality_p_df.append(pd.Series([p_shapiro, p_dagostino],
                                                     index=['Shapiro', 'DAgostino']),
                                           ignore_index=True)
    normality_stats_df = normality_stats_df.append(pd.Series([stat_shapiro, stat_dagostino],
                                                     index=['Shapiro', 'DAgostino']),
                                           ignore_index=True)
normality_p_df.index = samples_series
normality_stats_df.index = samples_series
display(normality_p_df)
print('P-values')
display(normality_stats_df)
print('Stats')<jupyter_output><empty_output><jupyter_text>Compute confidence intervals for each sampling numberWe will go with a 99% confidence<jupyter_code>import scipy.stats

def mean_confidence_interval(data, confidence=0.99):
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1)
    return h
ci_columns = ['f20', 'tnr', 'tpr_train', 'tpr_test']
ci_df = pd.DataFrame(columns=ci_columns)
means_df = pd.DataFrame(columns=sample_df.columns)

for sample_number in samples_series:

    samples_x_frames_df = sample_df[sample_df['#samples']==sample_number]
    ci_row = []
    for column in ci_columns:
        ci_row.append(mean_confidence_interval(samples_x_frames_df[column]))

    ci_df.loc[sample_number] = ci_row
    means_df.loc[sample_number] = samples_x_frames_df.mean()

means_df = means_df.drop(['#samples'], axis=1)
display(ci_df)
print('Confidence deltas')
display(means_df)
print('Metric means')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09833546734955186, 'fraction_numerical': 0.007618437900128041, 'mean_word_length': 3.171161548731642, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '611955', 'license': 'MIT', 'n_tokens_mistral': 4991, 'n_tokens_neox': 4734, 'n_words': 1215, 'path': 'machine_learning/notebooks/Confidence thresholds.ipynb', 'repo_name': 'cyberj0g/verification-classifier', 'size': '15643'}