<jupyter_start><jupyter_text>ANALYSING THE DATASET<jupyter_code>import pandas as pd
data = pd.read_csv("MalwareData.csv", sep="|")
data.head()
legitimate = data[0:41323].drop(["legitimate"], axis=1)
malware = data[41323:].drop(["legitimate"], axis=1)
print("Shape of legitimate dataset is %s samples with %s features" %(legitimate.shape[0], legitimate.shape[1]))
print("Shape of malware dataset is %s samples with %s features" %(malware.shape[0], malware.shape[1]))<jupyter_output>Shape of legitimate dataset is 41323 samples with 56 features
Shape of malware dataset is 96724 samples with 56 features<jupyter_text>CLASSIFIER<jupyter_code>from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_validate
data_input = data.drop(['Name', 'md5', 'legitimate', 'ResourcesMinSize', 'ResourcesMinEntropy', 'ResourcesMeanSize', 'ResourcesMaxSize', 'ResourcesMinSize', 'ResourcesMaxEntropy', 'ResourcesMinEntropy'], axis=1).values
labels = data["legitimate"].values
extratrees = ExtraTreesClassifier().fit(data_input, labels)
select = SelectFromModel(extratrees, prefit=True)
data_input_new = select.transform(data_input)
data_input_new
print(data_input.shape, data_input_new.shape)
import numpy as np
features = data_input_new.shape[1]
importances = extratrees.feature_importances_
indices = np.argsort(importances)[::-1]<jupyter_output><empty_output><jupyter_text>SELECTING THE MOST IMPORTANT FEATURES<jupyter_code>for x in range(features):
    print("%d"%(x+1), data.columns[2+indices[x]], importances[indices[x]])<jupyter_output>1 DllCharacteristics 0.15957226282291456
2 Machine 0.10294936670858645
3 Characteristics 0.10002110824166825
4 ResourcesMaxEntropy 0.07790531034994834
5 SectionsMaxEntropy 0.07650436744506255
6 MajorSubsystemVersion 0.0681976493822681
7 ImageBase 0.058449684791516215
8 Subsystem 0.05576108224155063
9 SizeOfOptionalHeader 0.051118315723415936
10 SizeOfStackReserve 0.0246548373016773
11 SectionsMeanEntropy 0.02130917062076368<jupyter_text>RANDOM FOREST CLASSIFIER<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
legitimate_train, legitimate_test, malware_train, malware_test = train_test_split(data_input_new, labels, test_size=0.2)
classifier = RandomForestClassifier(n_estimators=100)
classifier.fit(legitimate_train, malware_train)
print("Algorithm's score : ", classifier.score(legitimate_test, malware_test)*100)
from sklearn.metrics import confusion_matrix
result = classifier.predict(legitimate_test)
matrix = confusion_matrix(malware_test, result)
matrix
print("False Positives : ", matrix[0][1]/sum(matrix[0])*100)
print("False Negatives : ", matrix[1][0]/sum(matrix[1])*100)
import pickle
with open('model','wb') as file:
    pickle.dump(classifier, file)
import pickle
model_load = pickle.load(open('model','rb'))
import os
import pefile
import pandas as pd

def extract(path):
    pe = pefile.PE(path, fast_load=True)
    data = []
    entropy = list(map(lambda x:x.get_entropy(), pe.sections))
    SectionsMinEntropy = min(entropy)
    SectionsMaxEntropy = max(entropy)
    data.append(pe.OPTIONAL_HEADER.DllCharacteristics)
    data.append(pe.FILE_HEADER.Characteristics)
    data.append(pe.FILE_HEADER.Machine)
    data.append(pe.OPTIONAL_HEADER.ImageBase)
    data.append('6.9')
    data.append(SectionsMaxEntropy) 
    data.append(pe.OPTIONAL_HEADER.Subsystem)
    data.append(pe.FILE_HEADER.SizeOfOptionalHeader)
    data.append(pe.OPTIONAL_HEADER.MajorSubsystemVersion)
    data.append(SectionsMinEntropy)    
    data.append(pe.OPTIONAL_HEADER.MajorOperatingSystemVersion)
    data2 = []
    data2.append(data)
    return data2

attributes = extract('')
df = pd.DataFrame(attributes)
print(df.shape)
df1 = pd.DataFrame(data_input_new[4])
df1 = np.transpose(df1)
#print(df1.shape)
prediction = model_load.predict(df1)
print(prediction)
x=np.argmax(prediction.round(), axis=0)
if(x==0):
    print("LEGITIMATE")
else:
    print("MALWARE")<jupyter_output>LEGITIMATE<jupyter_text>GRADIENT BOOST CLASSIFIER<jupyter_code>from sklearn.ensemble import GradientBoostingClassifier
grad_boost = GradientBoostingClassifier(n_estimators=50)
grad_boost.fit(legitimate_train, malware_train)
print("Score of Gradient Boost Classifier : ", grad_boost.score(legitimate_test, malware_test)*100)
from sklearn.metrics import confusion_matrix
result = grad_boost.predict(legitimate_test)
matrix = confusion_matrix(malware_test, result)
print("False Positives : ", matrix[0][1]/sum(matrix[0])*100)
print("False Negatives : ", matrix[1][0]/sum(matrix[1])*100)
import pickle
with open('Grad_Boost_model','wb') as file:
    pickle.dump(grad_boost, file)
import os
import pefile
import pandas as pd

def extract(path):
    pe = pefile.PE(path, fast_load=True)
    data = []
    entropy = list(map(lambda x:x.get_entropy(), pe.sections))
    SectionsMinEntropy = min(entropy)
    SectionsMaxEntropy = max(entropy)
    data.append(pe.OPTIONAL_HEADER.DllCharacteristics)
    data.append(pe.FILE_HEADER.Characteristics)
    data.append(pe.FILE_HEADER.Machine)
    data.append(pe.OPTIONAL_HEADER.ImageBase)
    data.append('6.9')
    data.append(SectionsMaxEntropy) 
    data.append(pe.OPTIONAL_HEADER.Subsystem)
    data.append(pe.FILE_HEADER.SizeOfOptionalHeader)
    data.append(pe.OPTIONAL_HEADER.MajorSubsystemVersion)
    data.append(SectionsMinEntropy)    
    data.append(pe.OPTIONAL_HEADER.MajorOperatingSystemVersion)
    data2 = []
    data2.append(data)
    return data2

attributes = extract('')
df = pd.DataFrame(attributes)
print(df.shape)
df1 = pd.DataFrame(data_input_new[4])
df1 = np.transpose(df1)
#print(df1.shape)
prediction = grad_boost.predict(df1)
print(prediction)
x=np.argmax(prediction.round(), axis=0)
if(x==0):
    print("LEGITIMATE")
else:
    print("MALWARE")<jupyter_output>[1]
LEGITIMATE<jupyter_text>TESTING OTHER CLASSIFIERS<jupyter_code>from sklearn import tree, linear_model
import sklearn.ensemble as ek
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LinearRegression
model = { "DecisionTree":tree.DecisionTreeClassifier(max_depth=10),
         "RandomForest":ek.RandomForestClassifier(n_estimators=50),
         "Adaboost":ek.AdaBoostClassifier(n_estimators=50),
         "GradientBoosting":ek.GradientBoostingClassifier(n_estimators=50),
         "GNB":GaussianNB(),
         "LinearRegression":LinearRegression()   
        }
for x in model:
    classifier = model[x]
    classifier.fit(legitimate_train, malware_train)
    score = classifier.score(legitimate_test, malware_test)
    print(x+ ' '+ str(score))<jupyter_output>DecisionTree 0.9901847156827237
RandomForest 0.9939876856211518
Adaboost 0.9851865266207895
GradientBoosting 0.9889532777978993
GNB 0.7017385005432815
LinearRegression 0.5929045749430526
{'fraction_non_alphanumeric': 0.11059640995946729, 'fraction_numerical': 0.061088592935726695, 'mean_word_length': 6.8156108597285066, 'pattern_counts': {'":': 6, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '605685', 'license': 'MIT', 'n_tokens_mistral': 2635, 'n_tokens_neox': 2363, 'n_words': 404, 'path': 'Malware_Detection.ipynb', 'repo_name': 'InvincibleJuggernaut/Clamp', 'size': '6908'}