<jupyter_start><jupyter_text>Keras-Tensorflow Experiments  Sources: + https://www.tensorflow.org/tutorials/keras/overfit_and_underfit + https://www.kaggle.com/grafiszti/98-59-acc-on-10-fold-with-testing-7-keras-models + https://keras.io/visualization/ + https://stats.stackexchange.com/questions/181/how-to-choose-the-number-of-hidden-layers-and-nodes-in-a-feedforward-neural-netw  Initial Setup  Include needed files.<jupyter_code>from __future__ import absolute_import, division, print_function, unicode_literals

import shutil
import numpy as np
import pandas as pd
import tensorflow as tf

import csv
import os
import sys
import glob
import operator
import time

import seaborn as sn

import matplotlib.pyplot as plt

from keras.layers import Dense, Activation
from keras.utils.np_utils import to_categorical, normalize

from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.utils import shuffle
from sklearn.metrics import confusion_matrix

from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Dense, Activation, BatchNormalization, Dropout
from tensorflow.keras.callbacks import TensorBoard

print(tf.__version__)<jupyter_output>1.14.0<jupyter_text>Include Dataset<jupyter_code>%%bash
URL=https://iscxdownloads.cs.unb.ca/iscxdownloads/ISCX-URL-2016/
FILES=(ISCXURL2016.zip) 
for FILE in ${FILES[*]}; do
    if [ ! -f "$FILE" ]; then
        printf "downloading %s\n" $FILE
        curl -O $URL$FILE
        # unzip files
        echo 'unzipping ' $FILE
        unzip -o $FILE #overwrite exiting files/folders if exists
    fi
done<jupyter_output><empty_output><jupyter_text>Check Dataset<jupyter_code>! ls FinalDataset<jupyter_output>All_BestFirst.csv	      Malware_Infogain.csv
All_BestFirst_test.csv	      Malware_Infogain_test.csv
All.csv			      Phishing_BestFirst.csv
All.csv.pickle		      Phishing.csv
All_Infogain.csv	      Phishing_Infogain.csv
All_Infogain_test.csv	      Phishing_Infogain_test.csv
Defacement_BestFirst.csv      Spam_BestFirst.csv
Defacement.csv		      Spam_BestFirst_test.csv
Defacement_Infogain.csv       Spam.csv
Defacement_Infogain_test.csv  Spam_Infogain.csv
Malware_BestFirst.csv	      Spam_Infogain_test.csv
Malware.csv		      URL<jupyter_text>Set some data> Some data needs to be set, we need to ensure that constants are set properly. These are important but will not be used until later.<jupyter_code>resultPath = 'results_keras_tensorflow'
if not os.path.exists(resultPath):
   print('result path {} created.'.format(resultPath))
   os.mkdir(resultPath)
model_name = "init"<jupyter_output><empty_output><jupyter_text>Functions for Testing> Now that our data has been collected it is time to create functions that will be used in later tests.<jupyter_code>def loadData(csvFile):
    pickleDump = '{}.pickle'.format(csvFile)
    if os.path.exists(pickleDump):
        df = pd.read_pickle(pickleDump)
    else:
        df = pd.read_csv(csvFile, low_memory=False, na_values='NaN')
        # clean data
        # strip the whitspaces from column names
        df = df.rename(str.strip, axis='columns')
        #df.drop(columns=[], inplace=True)
        # drop missing values/NaN etc.
        #df.dropna(inplace=True)
        # drop Infinity rows and NaN string from each column
        for col in df.columns:
            indexNames = df[df[col]=='Infinity'].index
            if not indexNames.empty:
                print('deleting {} rows with Infinity in column {}'.format(len(indexNames), col))
                df.drop(indexNames, inplace=True)
        
        df.to_pickle(pickleDump)
    
    return df

def baseline_model(inputDim=-1,outputDim=-1):
    global model_extension, experimentTitle
    model = tf.keras.Sequential([
        Dense(128, activation='relu', input_shape=(inputDim,)),
        BatchNormalization(),
        Dropout(.5),
        Dense(64, activation='relu'),
        BatchNormalization(),
        Dropout(.5),
        Dense(outputDim, activation='softmax')
    ]) #This is the output layer

    if outputDim > 2:
        print('Categorical Cross-Entropy Loss Function')
        model_extension = "_categorical"
        experimentTitle = "Categorical"
        model.compile(optimizer='adam',
                 loss='categorical_crossentropy',
                 metrics=['accuracy'])
    else:
        print('Binary Cross-Entropy Loss Function')
        model_extension = "_binary"
        experimentTitle = "Binary"
        model.compile(optimizer='adam',
                loss='binary_crossentropy',
                metrics=['accuracy'])

    return model
def encode_labels(dataframe):
    dataframe=dataframe.copy()
    data_y=dataframe.pop(dep_var)
    encoder = LabelEncoder()
    encoder.fit(data_y)
    data_y = encoder.transform(data_y)
    dummy_y = to_categorical(data_y)
    return dummy_y
def plotAccuracy(title):
    plt.plot(history.history['acc'])
    plt.plot(history.history['val_acc'])
    plt.title(title)
    plt.ylabel('Accuracy')
    plt.xlabel('Epoch')
    plt.legend(['Train', 'Test'], loc='upper left')
    plt.show()
def plotLoss(title):
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title(title)
    plt.ylabel('Loss')
    plt.xlabel('Epoch')
    plt.legend(['Train', 'Test'], loc='upper left')
    plt.show()
def plot_Experiment(history,y):
  hist = pd.DataFrame(history.history)
  hist['epoch'] = history.epoch

  plt.figure()
  plt.xlabel('Epoch')
  plt.ylabel('Error')
  plt.plot(hist['epoch'], hist['loss'],
           label='Train Error')
  plt.plot(hist['epoch'], hist['val_loss'],
           label = 'Val Error')
  plt.ylim([0,y])
  plt.legend()

  plt.figure()
  plt.xlabel('Epoch')
  plt.ylabel('Error')
  plt.plot(hist['epoch'], hist['loss'],
           label='Train Error')
  plt.plot(hist['epoch'], hist['val_loss'],
           label = 'Val Error')
  plt.ylim([0,y])
  plt.legend()
  plt.show()
# Function to determine train and validation indexes, 
# and fit the data to the model we constructed
def experiment(dataframe, early=False):
    
    #10-fold cross validation, choosing random indices for training and validation
    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)
    
    # Encode the label column for model fitting
    encoded_y = dataframe.copy()
    encoded_y = encode_labels(encoded_y)
    
    # X is our data/features to train the model with
    X=StandardScaler().fit_transform(dataframe.drop(dep_var, axis=1))
    
    # Y is our labels to classify the data
    y=LabelEncoder().fit_transform(dataframe[dep_var].values)
    start_time = time.time()

    for index, (train_indices, val_indices) in enumerate(kfold.split(X, y)):
        
        xtrain, xval = X[train_indices], X[val_indices]
        ytrain, yval = encoded_y[train_indices], encoded_y[val_indices]

        inputDim=xtrain.shape[1]
        outputDim=ytrain.shape[1]
        print("Running fold #" + str(index+1))

        model = baseline_model(inputDim,outputDim)
        
        time_gen = int(time.time())
        
        global model_name
        model_name = f"{dataFile}_{time_gen}"

        tensorboard = TensorBoard(log_dir='keras_tensorflow_logs/{}/{}_{}'.format(experimentTitle, model_name, model_extension),update_freq='epoch')
        if early_stop:
            callbacks = [tensorboard, early_stop]
        else:
            callbacks = [tensorboard]
        history = model.fit(xtrain, ytrain, epochs=epochs, validation_data=(xval,yval), callbacks=callbacks, batch_size=batch_size, verbose=0)
        
    global end_time 
    end_time = time.time() - start_time
    TimeSetup = str(datetime.timedelta(seconds=end_time))
    Minutes = int(end_time/60)
    print("Time to complete {} [hour:min:sec]".format(TimeSetup)) 
        
    return model, history, X, encoded_y, TimeSetup<jupyter_output><empty_output><jupyter_text>Data Load and Clean> First we will load our data, scan the columns for Infinity and NaN values, and remove those columns from testing.<jupyter_code>df1 = loadData('FinalDataset/All.csv')
df1=df1.dropna(axis=1)<jupyter_output><empty_output><jupyter_text>Display Data Columns<jupyter_code>df1.columns<jupyter_output><empty_output><jupyter_text>Display Matrix Shape of Data> In the format (samples, columns)<jupyter_code>df1.shape<jupyter_output><empty_output><jupyter_text>Display First Samples in Data<jupyter_code>df1.head()<jupyter_output><empty_output><jupyter_text>Display Last Samples in Data<jupyter_code>df1.tail()<jupyter_output><empty_output><jupyter_text>Experimenting with Final Dataset/All.csv     Total Samples for each Type<jupyter_code>label = 'URL_Type_obf_Type'
lblTypes=set(df1[label])
for lbl in lblTypes:
    print('| {} | {} |'.format(lbl, len(df1[df1[label] == lbl].index)))
dataPath = 'FinalDataset'
dep_var = label<jupyter_output><empty_output><jupyter_text>Cast column values to float> Values in this column register as object type, which isn't valid for testing, so cast them to float<jupyter_code>df1.argPathRatio = df1['argPathRatio'].astype('float')<jupyter_output><empty_output><jupyter_text>Experimenting with Tensorflow Keras  Globals for Testing<jupyter_code>dataFile = 'All.csv'
epochs=100
batch_size=64<jupyter_output><empty_output><jupyter_text>Show uncoded label column<jupyter_code>df1[dep_var]<jupyter_output><empty_output><jupyter_text>Random seed for splitting<jupyter_code>seed = 7
np.random.seed(seed)
for file in glob.glob('keras_tensorflow_logs/Categorical*'):
        if os.path.exists(file):
            shutil.rmtree(file)
for file in glob.glob('keras_tensorflow_logs/Binary*'):
    if os.path.exists(file):
        shutil.rmtree(file)<jupyter_output><empty_output><jupyter_text>Overfitting and UnderfittingWe need to test the best way to avoid over/under-fitting. 1. I want to setup an early stop using the Keras Function2. I want to give the models a full train to see overfitting/underfitting results for the base model.2. I will test Early Stop to prevent underfitting3. I will test Weight Regularization4. I will test a dropout> our goal is to train quickly and efficiently.<jupyter_code>#Constants for this
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)
import datetime<jupyter_output><empty_output><jupyter_text>* We will use the function experiment and edit it until the model is properly fitted.   Model Creation steps1. Create a model with no changes2. Add the Early Stop3. test Dropout4. Test weight regularization5. Results  BaseLine test 1<jupyter_code>def TESTbaseline_model(inputDim=-1,outputDim=-1):
    global model_extension, experimentTitle
    model = tf.keras.Sequential([
        Dense(128, activation='relu', input_shape=(inputDim,)),
        BatchNormalization(),
        Dense(64, activation='relu'),
        BatchNormalization(),
        Dense(outputDim, activation='softmax')
    ]) #This is the output layer

    if outputDim > 2:
        print('Categorical Cross-Entropy Loss Function')
        model_extension = "_categorical"
        experimentTitle = "Categorical"
        model.compile(optimizer='adam',
                 loss='categorical_crossentropy',
                 metrics=['accuracy'])
    else:
        print('Binary Cross-Entropy Loss Function')
        model_extension = "_binary"
        experimentTitle = "Binary"
        model.compile(optimizer='adam',
                loss='binary_crossentropy',
                metrics=['accuracy'])

    return model
def BaseLine_experiment(dataframe, early=False):
    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)
    encoded_y = dataframe.copy()
    encoded_y = encode_labels(encoded_y)
    X=StandardScaler().fit_transform(dataframe.drop(dep_var, axis=1))
    y=LabelEncoder().fit_transform(dataframe[dep_var].values)
    start_time = time.time()
    for index, (train_indices, val_indices) in enumerate(kfold.split(X, y)):
        xtrain, xval = X[train_indices], X[val_indices]
        ytrain, yval = encoded_y[train_indices], encoded_y[val_indices]
        inputDim=xtrain.shape[1]
        outputDim=ytrain.shape[1]
        print("Running fold #" + str(index+1))
        model = TESTbaseline_model(inputDim,outputDim)    
        time_gen = int(time.time())        
        global model_name
        model_name = f"{dataFile}_{time_gen}"
        tensorboard = TensorBoard(log_dir='keras_tensorflow_logs/{}/{}_{}'.format(experimentTitle, model_name, model_extension),update_freq='epoch')
        if early_stop:
            callbacks = [tensorboard]
        else:
            callbacks = [tensorboard]
        history = model.fit(xtrain, ytrain, epochs=epochs, validation_data=(xval,yval), callbacks=callbacks, batch_size=batch_size, verbose=0)        
    global end_time 
    end_time = time.time() - start_time
    TimeSetup = str(datetime.timedelta(seconds=end_time))
    Minutes = int(end_time/60)
    print("Time to complete {} [hour:min:sec]".format(TimeSetup))    
    return model, history, X, encoded_y
categorical_df = df1.copy()
modelBL, BLhistory, BLX , BLencoded_y = BaseLine_experiment(categorical_df,early=True)
plot_Experiment(BLhistory,.5)
scores = modelBL.evaluate(BLX,BLencoded_y, batch_size=batch_size,verbose=1)
print(modelBL.metrics_names)
acc, loss = scores[1]*100, scores[0]
print('Baseline: accuracy: {:.2f}%: loss: {:.2f}'.format(acc, loss))<jupyter_output>36697/36697 [==============================] - 1s 27us/sample - loss: 0.0261 - acc: 0.9920
['loss', 'acc']
Baseline: accuracy: 99.20%: loss: 0.03<jupyter_text>Early Stop Implementation.<jupyter_code>def ESexperiment(dataframe, early=False):
    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)
    encoded_y = dataframe.copy()
    encoded_y = encode_labels(encoded_y)
    X=StandardScaler().fit_transform(dataframe.drop(dep_var, axis=1))
    y=LabelEncoder().fit_transform(dataframe[dep_var].values)
    start_time = time.time()
    for index, (train_indices, val_indices) in enumerate(kfold.split(X, y)):
        xtrain, xval = X[train_indices], X[val_indices]
        ytrain, yval = encoded_y[train_indices], encoded_y[val_indices]
        inputDim=xtrain.shape[1]
        outputDim=ytrain.shape[1]
        print("Running fold #" + str(index+1))
        model = TESTbaseline_model(inputDim,outputDim)    
        time_gen = int(time.time())        
        global model_name
        model_name = f"{dataFile}_{time_gen}"
        tensorboard = TensorBoard(log_dir='keras_tensorflow_logs/{}/{}_{}'.format(experimentTitle, model_name, model_extension),update_freq='epoch')
        if early_stop:
            callbacks = [tensorboard,early_stop]
        else:
            callbacks = [tensorboard, early_stop]
        history = model.fit(xtrain, ytrain, epochs=epochs, validation_data=(xval,yval), callbacks=callbacks, batch_size=batch_size, verbose=0)        
    global end_time 
    end_time = time.time() - start_time
    TimeSetup = str(datetime.timedelta(seconds=end_time))
    Minutes = int(end_time/60)
    print("Time to complete {} [hour:min:sec]".format(TimeSetup))    
    return model, history, X, encoded_y
categorical_df = df1.copy()
modelES, EShistory, ESX , ESencoded_y = ESexperiment(categorical_df,early=True)
plot_Experiment(EShistory,.5)
scores = modelES.evaluate(ESX,ESencoded_y, batch_size=batch_size,verbose=1)
print(modelES.metrics_names)
acc, loss = scores[1]*100, scores[0]
print('Baseline: accuracy: {:.2f}%: loss: {:.2f}'.format(acc, loss))<jupyter_output>36697/36697 [==============================] - 1s 32us/sample - loss: 0.0558 - acc: 0.9838
['loss', 'acc']
Baseline: accuracy: 98.38%: loss: 0.06<jupyter_text>Add Dropout<jupyter_code>def Dropoutbaseline_model(inputDim=-1,outputDim=-1):
    global model_extension, experimentTitle
    model = tf.keras.Sequential([
        Dense(128, activation='relu', input_shape=(inputDim,)),
        BatchNormalization(),
        Dropout(.5),
        Dense(64, activation='relu'),
        BatchNormalization(),
        Dropout(.5),
        Dense(outputDim, activation='softmax')
    ]) #This is the output layer

    if outputDim > 2:
        print('Categorical Cross-Entropy Loss Function')
        model_extension = "_categorical"
        experimentTitle = "Categorical"
        model.compile(optimizer='adam',
                 loss='categorical_crossentropy',
                 metrics=['accuracy'])
    else:
        print('Binary Cross-Entropy Loss Function')
        model_extension = "_binary"
        experimentTitle = "Binary"
        model.compile(optimizer='adam',
                loss='binary_crossentropy',
                metrics=['accuracy'])
def DPexperiment(dataframe, early=False):
    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)
    encoded_y = dataframe.copy()
    encoded_y = encode_labels(encoded_y)
    X=StandardScaler().fit_transform(dataframe.drop(dep_var, axis=1))
    y=LabelEncoder().fit_transform(dataframe[dep_var].values)
    start_time = time.time()
    for index, (train_indices, val_indices) in enumerate(kfold.split(X, y)):
        xtrain, xval = X[train_indices], X[val_indices]
        ytrain, yval = encoded_y[train_indices], encoded_y[val_indices]
        inputDim=xtrain.shape[1]
        outputDim=ytrain.shape[1]
        print("Running fold #" + str(index+1))
        model = TESTbaseline_model(inputDim,outputDim)    
        time_gen = int(time.time())        
        global model_name
        model_name = f"{dataFile}_{time_gen}"
        tensorboard = TensorBoard(log_dir='keras_tensorflow_logs/{}/{}_{}'.format(experimentTitle, model_name, model_extension),update_freq='epoch')
        if early_stop:
            callbacks = [tensorboard,early_stop]
        else:
            callbacks = [tensorboard, early_stop]
        history = model.fit(xtrain, ytrain, epochs=epochs, validation_data=(xval,yval), callbacks=callbacks, batch_size=batch_size, verbose=0)        
    global end_time 
    end_time = time.time() - start_time
    TimeSetup = str(datetime.timedelta(seconds=end_time))
    Minutes = int(end_time/60)
    print("Time to complete {} [hour:min:sec]".format(TimeSetup))    
    return model, history, X, encoded_y
categorical_df = df1.copy()
modelDropout, DropOuthistory, DPX , DPencoded_y = DPexperiment(categorical_df,early=True)
 
plot_Experiment(DropOuthistory,5)
scores = modelDropout.evaluate(DPX,DPencoded_y, batch_size=batch_size,verbose=1)
print(modelDropout.metrics_names)
acc, loss = scores[1]*100, scores[0]
print('Baseline: accuracy: {:.2f}%: loss: {:.2f}'.format(acc, loss))<jupyter_output>36697/36697 [==============================] - 1s 37us/sample - loss: 0.0426 - acc: 0.9862
['loss', 'acc']
Baseline: accuracy: 98.62%: loss: 0.04<jupyter_text>Add weight regularization<jupyter_code>def Regularize_Model(inputDim=-1,outputDim=-1):
  global model_extensionR, Experiment_TitleR
  model = tf.keras.Sequential([
        Dense(128, kernel_regularizer=keras.regularizers.l2(0.001), activation='relu',input_shape=(inputDim,)),
        BatchNormalization(),
        Dense(64, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001)),
        BatchNormalization(),
        Dense(outputDim, activation='softmax')
    ])
  if outputDim > 2:
        print('Categorical Cross-Entropy Loss Function')
        model_extension = "_categorical"
        experimentTitle = "Categorical"
        model.compile(optimizer='adam',
                 loss='categorical_crossentropy',
                 metrics=['accuracy'])
  else:
      print('Binary Cross-Entropy Loss Function')
      model_extension = "_binary"
      experimentTitle = "Binary"
      model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])
  return model
  
def L2experiment(dataframe, early=False):
    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)
    encoded_y = dataframe.copy()
    encoded_y = encode_labels(encoded_y)
    X=StandardScaler().fit_transform(dataframe.drop(dep_var, axis=1))
    y=LabelEncoder().fit_transform(dataframe[dep_var].values)
    start_time = time.time()
    for index, (train_indices, val_indices) in enumerate(kfold.split(X, y)):
        xtrain, xval = X[train_indices], X[val_indices]
        ytrain, yval = encoded_y[train_indices], encoded_y[val_indices]
        inputDim=xtrain.shape[1]
        outputDim=ytrain.shape[1]
        print("Running fold #" + str(index+1))
        model = Regularize_Model(inputDim,outputDim)    
        time_gen = int(time.time())        
        global model_name
        model_name = f"{dataFile}_{time_gen}"
        tensorboard = TensorBoard(log_dir='keras_tensorflow_logs/{}/{}_{}'.format(experimentTitle, model_name, model_extension),update_freq='epoch')
        if early_stop:
            callbacks = [tensorboard,early_stop]
        else:
            callbacks = [tensorboard, early_stop]
        history = model.fit(xtrain, ytrain, epochs=epochs, validation_data=(xval,yval), callbacks=callbacks, batch_size=batch_size, verbose=0)        
    global end_time 
    end_time = time.time() - start_time
    TimeSetup = str(datetime.timedelta(seconds=end_time))
    Minutes = int(end_time/60)
    print("Time to complete {} [hour:min:sec]".format(TimeSetup))    
    return model, history, X, encoded_y
categorical_df = df1.copy()
modelReg, Reghistory, RegX , Regencoded_y = L2experiment(categorical_df,early=True)
scores = modelReg.evaluate(RegX,Regencoded_y, batch_size=batch_size,verbose=1)
print(modelReg.metrics_names)
acc, loss = scores[1]*100, scores[0]
print('Baseline: accuracy: {:.2f}%: loss: {:.2f}'.format(acc, loss))<jupyter_output>36697/36697 [==============================] - 2s 43us/sample - loss: 0.1510 - acc: 0.9713
['loss', 'acc']
Baseline: accuracy: 97.13%: loss: 0.15<jupyter_text>Save Models<jupyter_code>modelReg.save('{}.model'.format("Regularization"))
modelDropout.save('{}.model'.format("DropOut"))
modelES.save('{}.model'.format("Early_Stop"))
modelBL.save('{}.model'.format("Base_Line"))<jupyter_output><empty_output><jupyter_text>Run the experiment<jupyter_code>categorical_df = df1.copy()
model, history, X , encoded_y, tTime = experiment(categorical_df,early=True)<jupyter_output>Running fold #1
Categorical Cross-Entropy Loss Function
Running fold #2
Categorical Cross-Entropy Loss Function
Running fold #3
Categorical Cross-Entropy Loss Function
Running fold #4
Categorical Cross-Entropy Loss Function
Running fold #5
Categorical Cross-Entropy Loss Function
Running fold #6
Categorical Cross-Entropy Loss Function
Running fold #7
Categorical Cross-Entropy Loss Function
Running fold #8
Categorical Cross-Entropy Loss Function
Running fold #9
Categorical Cross-Entropy Loss Function
Running fold #10
Categorical Cross-Entropy Loss Function
Time to complete 0:32:34.088277 [hour:min:sec]<jupyter_text>Save the model as a file<jupyter_code>model.save('{}.model'.format(os.path.basename(dataPath)))<jupyter_output><empty_output><jupyter_text>Model Results<jupyter_code>scores = model.evaluate(X,encoded_y, batch_size=batch_size,verbose=1)
print(model.metrics_names)
acc, loss = scores[1]*100, scores[0]
print('Baseline: accuracy: {:.2f}%: loss: {:.2f}'.format(acc, loss))<jupyter_output>36697/36697 [==============================] - 2s 45us/sample - loss: 0.1322 - acc: 0.9553
['loss', 'acc']
Baseline: accuracy: 95.53%: loss: 0.13<jupyter_text>Generate predictions from the model<jupyter_code>prediction_y = model.predict_classes(X, batch_size=batch_size, verbose=1)<jupyter_output>36697/36697 [==============================] - 3s 72us/sample<jupyter_text>Confusion Matrix<jupyter_code>y=LabelEncoder().fit_transform(categorical_df[dep_var].values)
cm = confusion_matrix(y, prediction_y)
sn.heatmap(cm, annot=True, fmt='g', cmap='Blues', xticklabels=['Defacement','Benign','Malware','Phishing','Spam'],
           yticklabels=['Defacement','Benign','Malware','Phishing','Spam'])<jupyter_output><empty_output><jupyter_text>Graph of Categorical Cross-Entropy Model<jupyter_code>plotAccuracy('Categorical Model Accuracy')
plotLoss('Categorical Model Loss')
plot_Experiment(history, 5)<jupyter_output><empty_output><jupyter_text>Write results to file<jupyter_code>def saveLog():
    resultFile = os.path.join(resultPath, dataFile)
    with open('{}.result'.format(resultFile), 'a') as fout:
        fout.write('{} results...'.format(model_name+model_extension))
        fout.write('\taccuracy: {:.2f} loss: {:.2f} time_elapsed: {} [hr:min:sec] \n'.format(acc, loss,tTime ))
saveLog()<jupyter_output><empty_output><jupyter_text>Binary Classification of Labels> Change all malicious labels to value 1 and benign label to 0  Binarize Data<jupyter_code>lblTypes = list(lblTypes)
lblTypes = dict(zip(lblTypes, [1, 1, 1, 1, 1]))
lblTypes['benign'] = 0
binary_df = df1.copy()
binary_df[label] = binary_df[label].map(lblTypes)<jupyter_output><empty_output><jupyter_text>Train the Binary Model<jupyter_code>model, history, X , encoded_y, tTime = experiment(binary_df)<jupyter_output>Running fold #1<jupyter_text>Model Results<jupyter_code>scores = model.evaluate(X,encoded_y, verbose=1)
print(model.metrics_names)
acc, loss = scores[1]*100, scores[0]
print('Baseline: accuracy: {:.2f}%: loss: {:.2f}'.format(acc, loss))<jupyter_output>36697/36697 [==============================] - 4s 115us/sample - loss: 0.0441 - acc: 0.9862
['loss', 'acc']
Baseline: accuracy: 98.62%: loss: 0.04<jupyter_text>Predictions<jupyter_code>prediction_y = model.predict_classes(X, batch_size=batch_size, verbose=1)
y=LabelEncoder().fit_transform(binary_df[dep_var].values)
cm = confusion_matrix(y, prediction_y)
sn.heatmap(cm, cmap='Blues', annot=True, fmt='g', xticklabels=['Benign', 'Malicious'],
        yticklabels=['Benign', 'Malicious'])<jupyter_output><empty_output><jupyter_text>Graph of Binary Cross-Entropy Model<jupyter_code>plotAccuracy('Binary Model Accuracy')
plotLoss('Binary Model Loss')<jupyter_output><empty_output><jupyter_text>Write Results to File<jupyter_code>saveLog()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12911518686209808, 'fraction_numerical': 0.016806077814442485, 'mean_word_length': 4.382693102023957, 'pattern_counts': {'":': 0, '<': 126, '<?xml version=': 0, '>': 137, 'https://': 5, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '34', 'dir': 'jupyter-structured-clean-dedup', 'id': '78587', 'license': 'MIT', 'n_tokens_mistral': 8904, 'n_tokens_neox': 8475, 'n_words': 1647, 'path': 'Keras-Tensorflow-Experiments.ipynb', 'repo_name': 'Shaurov05/DeepLearningMaliciousURLs', 'size': '26062'}