<jupyter_start><jupyter_text>Systems Security Project    Malware Classification using M<NAME>Dataset: Malware Images: Visualization and Automatic Classification.http://dl.acm.org/citation.cfm?id=2016908.![image-2.png](attachment:image-2.png)    Data: Jun/2021  Libraries<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import seaborn as sns
import os
from sklearn.model_selection import train_test_split
from sklearn.utils import class_weight
from sklearn.metrics import classification_report,confusion_matrix
from sklearn.metrics import mean_squared_error, roc_curve, auc
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, GlobalAveragePooling2D
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.preprocessing import image
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import SGD
import warnings
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Reading data<jupyter_code>#Generating DataSet 64 x 64 x 3
path_root = "malimg_paper_dataset_imgs\\"
    
batches = ImageDataGenerator().flow_from_directory(directory=path_root, target_size=(64,64), batch_size=10000)
imgs, labels = next(batches)
#Generating DataSet 150 x 150 x 3 for Inception V3

batches_resc = ImageDataGenerator().flow_from_directory(directory=path_root, target_size=(150,150), batch_size=10000)
imgs_resc, labels_resc = next(batches_resc)
families = [family for family in os.listdir(path_root)]
len (families)
print (imgs.shape)
print (labels.shape)
num_class = labels.shape
fig,ax = plt.subplots(1, figsize=(15,4)) # necessary to create an ax to plot labels
#plt.figure(figsize=(10,4)) 
plt.title('Images per class')
plt.bar(range(1,labels.shape[1]+1),labels.sum(axis=0))
plt.xticks(range (1,26), families, rotation=90)
for p in ax.patches:
        ax.annotate('{0}'.format(round(p.get_height(),0)), (p.get_x() + p.get_width()/2, p.get_height()), 
                    xytext=(0, 3),  textcoords="offset points", ha='center', va='bottom');<jupyter_output><empty_output><jupyter_text>Splitting data into train/ val and test sets<jupyter_code>#Split into train and test
X_train, X_aux, y_train, y_aux = train_test_split(imgs/255., labels, test_size=0.4, stratify=labels, random_state=1)
X_val, X_test, y_val, y_test = train_test_split(X_aux, y_aux, test_size=0.5, stratify=y_aux, random_state=1)
print (X_train.shape)
print (X_val.shape)
print (X_test.shape)
#Split into train and test - data augmentation
X_train_resc, X_aux_resc, y_train_resc, y_aux_resc = train_test_split(imgs_resc/255, labels_resc, test_size=0.4, stratify=labels_resc, random_state=1)

X_val_resc, X_test_resc, y_val_resc, y_test_resc = train_test_split(X_aux_resc, y_aux_resc, test_size=0.5, stratify=y_aux_resc, random_state=1)

print(X_train_resc.shape)
print (X_val_resc.shape)
print (X_test_resc.shape)<jupyter_output>(5603, 150, 150, 3)
(1868, 150, 150, 3)
(1868, 150, 150, 3)<jupyter_text>Dealing with unbalanced data (CNN)<jupyter_code>num_classes = 25 #rachel

y_train_new = np.argmax(y_train, axis=1)
print (y_train_new)

#Deal with unbalanced Data
weights = class_weight.compute_class_weight('balanced', np.unique(y_train_new), y_train_new)
weights_dict = dict(enumerate(weights))

print (weights)<jupyter_output>[10  2 24 ...  3  5  2]
[3.07013699 3.20171429 0.12669305 0.23492662 1.88336134 3.501875
 2.54681818 1.86766667 2.11433962 2.31051546 0.97868996 0.86868217
 1.7509375  2.03745455 3.02864865 2.35915789 2.73317073 2.63670588
 2.35915789 4.66916667 2.91064935 2.83696203 0.91477551 3.86413793
 0.46691667]<jupyter_text>Convolutional Neural Network (CNN) - Basic<jupyter_code>def Malware_model_CNN(num_classes):
    
    malware_model = Sequential()
    malware_model.add(Conv2D(30, kernel_size=(3, 3),
                     activation='relu',
                     input_shape=(64,64,3)))
    malware_model.add(MaxPooling2D(pool_size=(2, 2)))
    malware_model.add(Conv2D(15, (3, 3), activation='relu'))
    malware_model.add(MaxPooling2D(pool_size=(2, 2)))
    malware_model.add(Dropout(0.25))
    malware_model.add(Flatten())
    malware_model.add(Dense(128, activation='relu'))
    malware_model.add(Dropout(0.5))
    malware_model.add(Dense(50, activation='relu'))
    malware_model.add(Dense(num_classes, activation='softmax'))
    malware_model.compile(loss='categorical_crossentropy', optimizer = 'adam', metrics=['accuracy'])
    
    return malware_model<jupyter_output><empty_output><jupyter_text>Convolutional Neural Network (CNN) - Inceptionv3<jupyter_code>def Malware_model_InceptionV3(num_classes, X_train_resc, y_train_resc, X_test_resc, y_test_resc):

    # create the base pre-trained model
    base_model = InceptionV3(weights='imagenet', include_top=False)

    # add a global spatial average pooling layer
    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    # let's add a fully-connected layer
    x = Dense(1024, activation='relu')(x)
    # and a logistic layer
    predictions = Dense(num_classes, activation='softmax')(x)

    # this is the model we will train
    model = Model(inputs=base_model.input, outputs=predictions)

    # first: train only the top layers (which were randomly initialized)
    # i.e. freeze all convolutional InceptionV3 layers
    for layer in base_model.layers:
        layer.trainable = False

    # compile the model (should be done *after* setting layers to non-trainable)
    model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])

    # train the model on the new data for a few epochs
    model.fit(X_train_resc, y_train_resc, validation_data=(X_test_resc, y_test_resc), epochs=5, class_weight= weights_dict)

    # at this point, the top layers are well trained and we can start fine-tuning
    # convolutional layers from inception V3. We will freeze the bottom N layers
    # and train the remaining top layers.

    # let's visualize layer names and layer indices to see how many layers
    # we should freeze:
    for i, layer in enumerate(base_model.layers):
       print(i, layer.name)

    # we chose to train the top 2 inception blocks, i.e. we will freeze
    # the first 155 layers and unfreeze the rest:
    for layer in model.layers[:155]: 
       layer.trainable = False
    for layer in model.layers[155:]:
       layer.trainable = True

    # we need to recompile the model for these modifications to take effect
    # we use SGD with a low learning rate

    model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy'])

    # we train our model again (this time fine-tuning the top 2 inception blocks
    # alongside the top Dense layers
    model.fit(X_train_resc, y_train_resc, validation_data=(X_test_resc, y_test_resc), epochs=10, class_weight= weights_dict)
    
    return model<jupyter_output><empty_output><jupyter_text>Training the model and test<jupyter_code>#Train and test our model                        
model_cnn = Malware_model_CNN(num_classes)
model_cnn.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=10, class_weight= weights_dict)
scores_cnn = model_cnn.evaluate(X_test, y_test)
predict_train_cnn = model_cnn.predict(X_train)
predict_test_cnn = model_cnn.predict(X_test)
model_inc_v3 = Malware_model_InceptionV3(num_classes, X_train_resc, y_train_resc, X_val_resc, y_val_resc)
scores_inc_v3 = model_inc_v3.evaluate(X_test_resc, y_test_resc)
predict_train_inc_v3 = model_inc_v3.predict(X_train_resc)
predict_test_inc_v3 = model_inc_v3.predict(X_test_resc)<jupyter_output><empty_output><jupyter_text>Evaluation the Algorithms **Confusion matrix:**  - True positives: positive tuples correctly labeled   - True negatives: negative tuples correctly labeled   - False positives: negative tuples incorrectly labeled   - False negatives: positive tuples incorrectly labeled  **Classification report:**   - Accuracy = (TP+TN)/(TP+TN+FP+FN)   - Precision = TP/(TP+FP)   - Recall = TP/(TP+FN)   - F1 - measure = 2rp/(r+p)   - WeightedAccuracy = (Wtp*TP+Wtn*TN)/(Wtp*TP+Wtn*TN+Wfp*FP+Wfn*FN)  **Error:**- Mean squared error (MSE)**Receiver Operating Characteristic (ROC) curve:**  - Area Under the Curve (AUC)<jupyter_code>def print_confusion_matrix (y_test, predict_test, families):

    print ('Confusion matrix:')
    matrix = confusion_matrix(y_test.argmax(axis=1), predict_test.argmax(axis=1)) # argmax(axis=1) Get most likely class

    fig, ax = plt.subplots(figsize=(10,10)) 
    sns.heatmap(matrix, annot=True, fmt="d", ax = ax, vmin=0, vmax=100) 
    plt.xticks(range (1,26), families, rotation=90)
    plt.yticks(range (1,26), families, rotation=0);
    
def print_classification_report (y_test, predict_test, families):
    
    print(f'Classification Report:')
    print(classification_report(y_test.argmax(axis=1), predict_test.argmax(axis=1), target_names=families))
    print ('MSE:', round(mean_squared_error (y_test,predict_test),3))
    

def print_roc_curve (y_test, predict_test):
    
    # Compute ROC curve and ROC area for each class
    for i in range (0,25):
        fpr, tpr, thresh = roc_curve(y_test[:,i], predict_test[:,i])
        area = auc(fpr, tpr)
        plt.plot(fpr, tpr, label='%s (area = %.2f)' % (families[i], area))
    
    plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', label='Random guess')
    plt.title(f'AUC & ROC curve')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.grid()
    plt.show
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))<jupyter_output><empty_output><jupyter_text>Evaluation - Basic CNN<jupyter_code>print_confusion_matrix (y_test, predict_test_cnn, families)
print_classification_report (y_test, predict_test_cnn, families)
print_roc_curve (y_test, predict_test_cnn)<jupyter_output><empty_output><jupyter_text>Evaluation - Inceptionv3<jupyter_code>print_confusion_matrix (y_test_resc, predict_test_inc_v3, families)
print_classification_report (y_test_resc, predict_test_inc_v3, families)
print_roc_curve (y_test_resc, predict_test_inc_v3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11038262060867013, 'fraction_numerical': 0.04589490165378217, 'mean_word_length': 5.4035087719298245, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '251349', 'license': 'MIT', 'n_tokens_mistral': 3926, 'n_tokens_neox': 3596, 'n_words': 798, 'path': '10_systems_security/malware_project_oficial.ipynb', 'repo_name': 'rachelfanti/data_science', 'size': '10238'}