<jupyter_start><jupyter_text>__Delete the train.7z file__<jupyter_code>import warnings
warnings.filterwarnings("ignore")
import shutil
import os
import pandas as pd
import matplotlib
matplotlib.use(u'nbAgg')
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pickle
from sklearn.manifold import TSNE
from sklearn import preprocessing
import pandas as pd
from multiprocessing import Process
import multiprocessing
import codecs # this is used for file operations 
import random as r
from xgboost import XGBClassifier
from sklearn.model_selection import RandomizedSearchCV
from sklearn.tree import DecisionTreeClassifier
from sklearn.calibration import CalibratedClassifierCV
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import log_loss
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier<jupyter_output><empty_output><jupyter_text>Distribution of malware classes in whole data set<jupyter_code>%matplotlib inline

Y = pd.read_csv("trainLabels.csv")
total = len(Y)*1.
ax=sns.countplot(x="Class", data=Y)
for p in ax.patches:
        ax.annotate('{:.1f}%'.format(100*p.get_height()/total), (p.get_x()+0.1, p.get_height()+5))

#put 11 ticks (therefore 10 steps), from 0 to the total number of rows in the dataframe
ax.yaxis.set_ticks(np.linspace(0, total, 11))

#adjust the ticklabel to the desired format, without changing the position of the ticks. 
ax.set_yticklabels(map('{:.1f}%'.format, 100*ax.yaxis.get_majorticklocs()/total))
plt.show()
print(Y.shape)

print('-' * 100)

print(Y.head())<jupyter_output>(10868, 2)
----------------------------------------------------------------------------------------------------
                     Id  Class
0  01kcPWA9K2BOxQeS5Rju      1
1  04EjIdbPV5e1XroFOpiN      1
2  05EeG39MTRrI6VY21DPd      1
3  05rJTUWYAKNegBk2wE8X      1
4  0AnoOZDNbPXIr2MRBSCJ      1<jupyter_text>Feature Extraction<jupyter_code>#file sizes of byte files

files=os.listdir('bytes')
filenames=Y['Id'].tolist()
class_y=Y['Class'].tolist()
class_bytes=[]
sizebytes=[]
fnames=[]

for file in files:
    # print(os.stat('byteFiles/0A32eTdBKayjCWhZqDOQ.txt'))
    # os.stat_result(st_mode=33206, st_ino=1125899906874507, st_dev=3561571700, st_nlink=1, st_uid=0, st_gid=0, 
    # st_size=3680109, st_atime=1519638522, st_mtime=1519638522, st_ctime=1519638522)
    # read more about os.stat: here https://www.tutorialspoint.com/python/os_stat.htm
    statinfo=os.stat('bytes/'+file)
    # split the file name at '.' and take the first part of it i.e the file name
    file=file.split('.')[0]
    if any(file == filename for filename in filenames):
        i=filenames.index(file)
        class_bytes.append(class_y[i])
        # converting into Mb's
        sizebytes.append(statinfo.st_size/(1024.0*1024.0))
        fnames.append(file)
data_size_byte=pd.DataFrame({'ID':fnames,'size':sizebytes,'Class':class_bytes})
print(data_size_byte.head())
data_size_byte.shape
data_size_byte.to_csv('train_labels_with_byte_size.csv')<jupyter_output><empty_output><jupyter_text>Extracting Bigrams from Byte files<jupyter_code>import itertools

hex_code_ = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']

bigrams_unique = itertools.product(hex_code_, repeat=4)
bigrams_unique = [''.join(map(str, i)) for i in list(bigrams_unique)]
len(bigrams_unique)
#removal of addres from byte files
# contents of .byte files
# ----------------
#00401000 56 8D 44 24 08 50 8B F1 E8 1C 1B 00 00 C7 06 08 
#-------------------
#we remove the starting address 00401000

files = os.listdir('bytes')
filenames=[]
array=[]
for file in files:
    if(file.endswith("bytes")):
        file=file.split('.')[0]
        text_file = open('bytes/'+file+".txt", 'w+')
        with open('bytes/'+file+".bytes","r") as fp:
            for line in fp:
                put_space = False
                output_line = ''
                for idx, ele in enumerate(line.rstrip().split(" ")[1:]):
                    if (idx+1) % 2 == 0:
                        output_line += ele + ' '
                    else:
                        output_line += ele
                output_line = output_line + "\n"
                text_file.write(output_line)
            fp.close()
            os.remove('bytes/'+file+".bytes")
        text_file.close()
files = os.listdir('bytes')
filenames2=[]
feature_matrix = np.zeros((len(files),len(bigrams_unique)+1),dtype=int)
k = 0

#program to convert into bag of words of bytefiles
#this is custom-built bag of words this is bigram bag of words
byte_feature_file=open('bytes_feature_output.csv','w+')
byte_feature_file.write(','.join(bigrams_unique) + ',????')
byte_feature_file.write("\n")
for file in files:
    filenames2.append(file)
    byte_feature_file.write(file+",")
    if(file.endswith("txt")):
        with open('bytes/'+file,"r") as byte_flie:
            for lines in byte_flie:
                line=lines.rstrip().split(" ")
                for hex_code in line:
                    if hex_code=='????':
                        feature_matrix[k][len(bigrams_unique)]+=1
                    elif '?' in hex_code:
                        pass
                    else:
                        feature_matrix[k][int(hex_code,16)]+=1
        byte_flie.close()
    for i, row in enumerate(feature_matrix[k]):
        if i!=len(feature_matrix[k])-1:
            byte_feature_file.write(str(row)+",")
        else:
            byte_feature_file.write(str(row))
    byte_feature_file.write("\n")
    
    k += 1

byte_feature_file.close()
# Removing all byte files
import shutil

shutil.rmtree('bytes')
!cp bytes_feature_output.csv '/content/drive/My Drive/Datasets'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14449580551275468, 'fraction_numerical': 0.036637562061290874, 'mean_word_length': 3.1818181818181817, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '441108', 'license': 'MIT', 'n_tokens_mistral': 2145, 'n_tokens_neox': 1974, 'n_words': 447, 'path': 'byte_files_processing/Microsoft__Malware_Detection_FE_Bytes_file.ipynb', 'repo_name': 'd-Gaurav/Microsoft-Malware-Detection', 'size': '5841'}