<jupyter_start><jupyter_text>The code in this notebook is entirely for making example plots to use in the technical background of the report   This notebook doesn't have any relation to the overall firewall project<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression<jupyter_output><empty_output><jupyter_text>Logistic regression<jupyter_code>xmin, xmax = -7, 5
n = 50
np.random.seed(0)
X = np.random.normal(size=n)
y = (X > 0).astype(np.float)
X[X >= 0] *= 4
X[X >= 0] += 2
X[X < 0] -= 2

# plot separated
f, axarr = plt.subplots(1,2, sharey=True)
f.set_figheight(5)
f.set_figwidth(12)

'''
Figure 0
'''
for t,marker,c in zip([0.0,1.0],"ox","gb"):
    # plot each class on its own to get different colored markers
    axarr[0].scatter(X[y == t],
                np.zeros(len(X[y==t])),
                marker=marker,
                c=c)
    
X_test = np.linspace(-7, 10, 300)

axarr[0].set_ylabel('y')
axarr[0].set_xlabel('X')
axarr[0].set_xticks(range(-7, 10))
axarr[0].set_yticks([0, 0.5, 1])
axarr[0].set_ylim(-.25, 1.25)
axarr[0].set_xlim(-7, 10)
axarr[0].legend(('class 0', 'class 1'),
           loc="lower right", fontsize='small')


'''
Figure 1
'''
for t,marker,c in zip([0.0,1.0],"ox","gb"):
    # plot each class on its own to get different colored markers
    axarr[1].scatter(X[y == t],
                y[y==t],
                marker=marker,
                c=c)
    
#plt.scatter(X, y, c=y,color='black', zorder=20)
X_test = np.linspace(-7, 10, 300)


def model(x):
    return 1 / (1 + np.exp(-x))

loss = model(X_test)
axarr[1].plot(X_test, loss, color='red', linewidth=3)


axarr[1].set_ylabel('y')
axarr[1].set_xlabel('X')
axarr[1].set_xticks(range(-7, 10))
axarr[1].set_yticks([0, 0.5, 1])
axarr[1].set_ylim(-.25, 1.25)
axarr[1].set_xlim(-7, 10)
axarr[1].legend(('Logistic Regression Model', 'class 0', 'class 1'),
           loc="lower right", fontsize='small')
plt.show()

f.savefig('images/report_images/logistic.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Support vector machineThis is matlab code on how the image demonstrating linear vs RBF kernel was made (svm_kernel.png)<jupyter_code>'''
clear
clc
load('d2.mat');

hold on
gscatter(X(:,1),X(:,2),Y,'rb','x+',6);

SVMstruct = svmtrain(X,Y,'boxconstraint',1,'autoscale',false,'kernel_function','RBF');
        
% Make a grid of values to classify the entire space
x1_axis = linspace(min(X(:,1)), max(X(:,1)), 1000)';
x2_axis = linspace(min(X(:,2)), max(X(:,2)), 1000)';

[x1_space, x2_space] = meshgrid(x1_axis, x2_axis);

for i = 1:size(x1_space, 2)
   point_in_space = [x1_space(:, i), x2_space(:, i)];
   class(:, i) = svmclassify(SVMstruct, point_in_space);
end

% Plot the SVM boundary
hold on
contour(x1_space, x2_space, class, [0 0], 'k');
legend('-1','1','RBF boundary');
xlabel('x1');
ylabel('x2');
title('Data points and decision boundary');
hold off;
'''
import pickle
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier

def get2Grams(payload_obj):
    '''Divides a string into 2-grams
    
    Example: input - payload: "<script>"
             output- ["<s","sc","cr","ri","ip","pt","t>"]
    '''
    payload = str(payload_obj)
    ngrams = []
    for i in range(0,len(payload)-2):
        ngrams.append(payload[i:i+2])
    return ngrams

classifier = pickle.load( open("data/tfidf_2grams_randomforest.p", "rb"))

def injection_test(inputs):
    variables = inputs.split('&')
    values = [ variable.split('=')[1] for variable in variables]
    print(values)
    return 'MALICIOUS' if classifier.predict(values).sum() > 0 else 'NOT_MALICIOUS'

injection_test('var1=<rip cookie')
def get1Grams(payload_obj):
    '''Divides a string into 1-grams
    
    Example: input - payload: "<script>"
             output- ["<","s","c","r","i","p","t",">"]
    '''
    payload = str(payload_obj)
    ngrams = []
    for i in range(0,len(payload)-1):
        ngrams.append(payload[i:i+1])
    return ngrams

def get2Grams(payload_obj):
    '''Divides a string into 2-grams
    
    Example: input - payload: "<script>"
             output- ["<s","sc","cr","ri","ip","pt","t>"]
    '''
    payload = str(payload_obj)
    ngrams = []
    for i in range(0,len(payload)-2):
        ngrams.append(payload[i:i+2])
    return ngrams

def get3Grams(payload_obj):
    '''Divides a string into 3-grams
    
    Example: input - payload: "<script>"
             output- ["<sc","scr","cri","rip","ipt","pt>"]
    '''
    payload = str(payload_obj)
    ngrams = []
    for i in range(0,len(payload)-3):
        ngrams.append(payload[i:i+3])
    return ngrams

classifierz = pickle.load( open("data/trained_classifiers.p", "rb"))[['accuracy','sensitivity','specificity','auc','conf_matrix','params']]

from IPython.display import display
import pandas as pd
display(classifierz)
classifierz.to_csv('data/classifiers_result_table.csv',encoding='UTF-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1806529140797116, 'fraction_numerical': 0.031243741237732825, 'mean_word_length': 4.049544994944388, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '628010', 'license': 'MIT', 'n_tokens_mistral': 1983, 'n_tokens_neox': 1874, 'n_words': 405, 'path': 'Plots_technical_background.ipynb', 'repo_name': 'grananqvist/Machine-Learning-Web-Application-Firewall-and-Datase', 'size': '4993'}