<jupyter_start><jupyter_text>Malware Detector: Testing and Tuning the ModelFeatures by Bhat's AnalysisFinal Model: Final_Malware_Detector.ipynb<jupyter_code>import pandas as pd
import numpy as np
data= pd.read_csv('/data/top_1000_pe_imports.csv')
df= data[['OpenProcess', 'GetCurrentProcess', 'GetProcessHeap','ReadFile', 'CreateFileW', 'WriteFile', 'FindFirstFileW', 'FindNextFileW', 'SetWindowsHookExW', 'GetAsyncKeyState', 'GetForegroundWindow', 'GetKeyState', 'MapVirtualKeyW', 'VirtualAlloc', 'VirtualProtect', 'GetModuleHandleA', 'ExitProcess', 'RegCloseKey','GetCurrentProcessId', 'malware']]
#df.columns.to_list()
y= df['malware']
X= df.drop(['malware'], axis=1)
print(X.shape, y.shape)
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
X_train, X_test, y_train, y_test= train_test_split(X, y, test_size=0.2, random_state=2)
#print(X_train.shape, X_test.shape)
#print(y_train.shape, y_test.shape)
#Random Forest Classifier

rf = RandomForestClassifier(n_estimators=50, random_state=3)
rf.fit(X_train,y_train)<jupyter_output><empty_output><jupyter_text>Evaluation of Model<jupyter_code>from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, f1_score
from sklearn.model_selection import cross_val_score, cross_val_predict

y_pred= rf.predict(X_test)

print("Train Accuracy:",rf.score(X_train, y_train) )
print( "Test Accuracy:", accuracy_score(y_test, y_pred, normalize=True))
#since accuracy is pretty high (97.8%), will check on cross validation

scores= cross_val_score(rf, X_train, y_train, cv=5, scoring='accuracy') #5 fold

print("Scores: ", scores)
print("Mean: ", scores.mean())
print("Standard Deviation: ", scores.std())
#Still pretty good accuracy, so checking Out of Bag score

rf = RandomForestClassifier(n_estimators=50, oob_score = True, random_state=3)
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)

print("OOB score:", round(rf.oob_score_, 4)*100, "%")
def evaluation(model, X_val, y_val, predictions):
    print("Confusion Matrix: \n", confusion_matrix(y_val, predictions))
    print("Precision:", precision_score(y_val, predictions))
    print("Recall:",recall_score(y_val, predictions))
    print("F1 Score:", f1_score(y_val, predictions))
    return
#Cross Val Evaluation

print("Cross Validation:\n")
predictions = cross_val_predict(rf, X_train, y_train, cv=3)
evaluation(rf, X_train, y_train, predictions)


#Confusion matrix: [[TN, FP], [FN,TP]]
#Testing Evaluation

print("Test Set:\n")
y_pred = rf.predict(X_test)
evaluation(rf, X_test, y_test, y_pred)<jupyter_output>Test Set:

Confusion Matrix: 
 [[ 201  196]
 [  44 9075]]
Precision: 0.9788588070326826
Recall: 0.9951749095295537
F1 Score: 0.9869494290375205<jupyter_text>High Precision= Low false positive rateHigh Recall= Low false negative rateSo I am seeing some false positives. But good news is that we have VVV high true positives and VVV low false negatives, which is required for a Malware Detector  Feature Importances<jupyter_code>#Checking what Features are deamed important by the classifier; cross check with how it relates to Bhat's Stuff

importances= pd.DataFrame({'Feature': X_train.columns, 'Importance': np.round(rf.feature_importances_,3)})
importances= importances.sort_values('Importance', ascending=False).set_index('Feature')

importances<jupyter_output><empty_output><jupyter_text>Hyperparameter Testing<jupyter_code># Current parameters in use
from pprint import pprint
# Look at parameters used by our current forest
print('Parameters currently in use:\n')
pprint(rf.get_params())<jupyter_output>Parameters currently in use:

{'bootstrap': True,
 'ccp_alpha': 0.0,
 'class_weight': None,
 'criterion': 'gini',
 'max_depth': None,
 'max_features': 'auto',
 'max_leaf_nodes': None,
 'max_samples': None,
 'min_impurity_decrease': 0.0,
 'min_impurity_split': None,
 'min_samples_leaf': 1,
 'min_samples_split': 2,
 'min_weight_fraction_leaf': 0.0,
 'n_estimators': 50,
 'n_jobs': None,
 'oob_score': True,
 'random_state': 3,
 'verbose': 0,
 'warm_start': False}<jupyter_text>Parameters I want to work on:1. bootstrap = method for sampling data points (with or without replacement) (False= entire dataset used to make trees)2. max_depth = max number of levels in each decision tree  (More depth may lead to single leaf nodes in each tree, less depth may not be sufficient enough to capture the info)3. max_features = max number of features considered for splitting a node4. n_estimators = number of trees in the forest 5. min_samples_split = min number of data points placed in a node before the node is split (how many samples are needed to split the node)6. min_samples_leaf = min number of data points allowed in a leaf node (helps smoothen the model)<jupyter_code>#Testing diff values: each iteration, algo chooses a different combination of features
from sklearn.model_selection import RandomizedSearchCV

n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]
max_features = ['auto', 'sqrt']
max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
max_depth.append(None)
min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}
pprint(random_grid)


rf = RandomForestClassifier() #Base Model
rf_random = RandomizedSearchCV(estimator = rf, 
                               param_distributions = random_grid, 
                               n_iter = 100, 
                               cv = 3, 
                               verbose=2, 
                               random_state=42,
                               n_jobs = -1)

# 3 folds of cross validation, more cv reduces overfitting but increases run time.
# n_jobs=-1 uses all processors in parallel
# Verbosity helps log the output. https://stats.stackexchange.com/questions/153823/what-is-verbose-in-scikit-learn-package-of-python

rf_random.fit(X_train, y_train)
#Show best parameters
rf_random.best_params_<jupyter_output><empty_output><jupyter_text>Found Best Parameters:1. n_estimators: 10002. min_smaples_split: 53. min_samples_leaf: 24. max_features: sqrt5. max_depth: 1006. bootstrap: False<jupyter_code>from sklearn.model_selection import GridSearchCV
# Create the parameter grid based on the results of random search 
param_grid = {
    'bootstrap': [False],
    'max_depth': [100, 110],
    'max_features': [2, 3],
    'min_samples_leaf': [1, 2, 3],
    'min_samples_split': [3, 5, 7],
    'n_estimators': [800, 900, 100]
}
# Create a based model
rf = RandomForestClassifier()
# Instantiate the grid search model
grid_search = GridSearchCV(estimator = rf, param_grid = param_grid, 
                          cv = 3, n_jobs = -1, verbose = 2)


grid_search.fit(X_train, y_train)
grid_search.best_params_<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10961429381735678, 'fraction_numerical': 0.02694271128757799, 'mean_word_length': 5.0540772532188845, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '195469', 'license': 'MIT', 'n_tokens_mistral': 2465, 'n_tokens_neox': 2281, 'n_words': 638, 'path': 'code/Model_Tuning.ipynb', 'repo_name': 'aashka-trivedi/malware-detector', 'size': '7052'}