<jupyter_start><jupyter_text><jupyter_code>!git clone https://github.com/aftab-ali/Tensorflow2.0.git
%cd Tensorflow2.0/
import numpy
import os
import  joblib
import pickle
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.preprocessing.text import Tokenizer
from keras.layers import LSTM, Dense, Dropout, Embedding
from keras.preprocessing import sequence
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import SpatialDropout1D
from keras.models import model_from_json
from mlxtend.plotting import plot_confusion_matrix
malware_calls_df = pd.read_csv("calls.zip", compression="zip",
                               sep="\t", names=["API_Calls"])

malware_labels_df = pd.read_csv("types.zip", compression="zip",
                                sep="\t", names=["API_Labels"])

malware_calls_df["API_Labels"] = malware_labels_df.API_Labels
malware_calls_df["API_Calls"] = malware_calls_df.API_Calls.apply(lambda x: " ".join(x.split(",")))

malware_calls_df["API_Labels"] = malware_calls_df.API_Labels.apply(lambda x: 1 if x == "Virus" else 0)
sns.countplot(malware_calls_df.API_Labels)
plt.xlabel('Labels')
plt.title('Class Distribution')
plt.savefig("class_distribution.png")
plt.show()
max_words = 800
max_len = 100

X = malware_calls_df.API_Calls
Y = malware_calls_df.API_Labels.astype('category').cat.codes

tok = Tokenizer(num_words=max_words)
tok.fit_on_texts(X)
print('Found %s unique tokens.' % len(tok.word_index))
X = tok.texts_to_sequences(X.values)
X = sequence.pad_sequences(X, maxlen=max_len)
print('Shape of data tensor:', X.shape)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

le = LabelEncoder()
Y_train_enc = le.fit_transform(Y_train)
Y_train_enc = np_utils.to_categorical(Y_train_enc)

Y_test_enc = le.transform(Y_test)
Y_test_enc = np_utils.to_categorical(Y_test_enc)
# Sequential Model
# activation_function = 'softsign'
def malware_model(act_func="softsign"):
    model = Sequential()
    model.add(Embedding(max_words, 300, input_length=max_len))
    model.add(SpatialDropout1D(0.1))
    model.add(LSTM(32, dropout=0.1, recurrent_dropout=0.1, return_sequences=True, activation=act_func))
    model.add(LSTM(32, dropout=0.1, activation=act_func, return_sequences=True))
    model.add(LSTM(32, dropout=0.1, activation=act_func))
    model.add(Dense(128, activation=act_func))
    model.add(Dropout(0.1))
    model.add(Dense(256, activation=act_func))
    model.add(Dropout(0.1))
    model.add(Dense(128, activation=act_func))
    model.add(Dropout(0.1))
    model.add(Dense(1, name='out_layer', activation="linear"))
    return model
class myCallback(tf.keras.callbacks.Callback):
  def on_epoch_end(self, epoch, logs={}):
    if(logs.get('accuracy') > 0.90):
      print("\nReached +90% Accuracy so cancelling Training!")
      self.model.stop_training = True

callbacks = myCallback()
model = malware_model()
print(model.summary())
model.compile(loss='mse', optimizer="rmsprop", metrics=['accuracy'])

# history
history = model.fit(X_train, Y_train, batch_size=1000, epochs=10,
                    verbose=1, callbacks=[callbacks], 
                    validation_data=(X_test, Y_test))

# validation_data=(X_test, Y_test)
# evaluate the model
print(X_train.shape)
print(X_test.shape)
scores = model.evaluate(X_test, Y_test, verbose=1)
# metics_names = loss, accuracy
# scores = returns (loss_percentage accuracy_percentage)
# %s (string), %.2f (02 digits floating point precision)
print("%s: %.2f" % (model.metrics_names[1].title(), scores[1]*100))
# Saving Model to a JSON file and it's corresponding weights
# Serializing model to JSON
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)
# Serializing Weights to HDF5
model.save_weights("model.h5")
print("Saved model to disk")
# Load JSON and create model
json_file = open('model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)

# Load Weights Into New Model
loaded_model.load_weights("model.h5")
print("Loaded model from disk")
# 20% is 1422
# Evaluating loaded model on Test Data
loaded_model.compile(loss='mse', optimizer='rmsprop', metrics=['accuracy'])
score = loaded_model.evaluate(X_test, Y_test, verbose=0)
print("%s: %.2f" % (loaded_model.metrics_names[1].title(), score[1]*100))
classification = model.predict(X_test)
print(classification[100])
y_test_pred = model.predict_classes(X_test)
cm = confusion_matrix(Y_test, y_test_pred)

plot_confusion_matrix(conf_mat=cm,
                      show_absolute=True,
                      show_normed=True,
                      colorbar=True)
plt.savefig("Confusion_Matrix.png")
plt.show()  
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.grid()
plt.savefig("accuracy.png")
plt.show()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.grid()
plt.savefig("loss.png")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13579335793357933, 'fraction_numerical': 0.016974169741697416, 'mean_word_length': 5.547101449275362, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '27057', 'license': 'MIT', 'n_tokens_mistral': 1981, 'n_tokens_neox': 1885, 'n_words': 366, 'path': 'LSTM_Malware_Detection.ipynb', 'repo_name': 'aftab-ali/Tensorflow2.0', 'size': '5420'}