<jupyter_start><jupyter_text>Gradient Boosting Classifier on DARKNET 2020: A Project Report<jupyter_code>#importing python libraries
import numpy as np
import pandas as pd
#Reading the dataset: Darknet 2020
df = pd.read_csv('darknet.csv', error_bad_lines=False)
df.columns = df.columns.str.strip()
df.head(10)
#A look at all the 85 columns of the dataset
df.columns
#Getting all the columns with total number of entries in each column and data type
df.info()
#Correlation matrix to know the dependency of columns on each other
corr = df.corr()
corr.head()
df.dropna() #dropping null values
df.isnull().sum().head() #to check the number of null values in each column
df['Label'].value_counts()  #to check the number of classes in Label
df['Label.1'].value_counts() #to check the number of classes in Label
#splitting the Src IP into octets,getting first two ocets
newIP = []
for value in df['Src IP']:
    IP = value.split(".")
    octet1= IP[0]
    octet2= IP[1]
#     print(octet2)
    newIP.append(float(octet1 + '.' + octet2))

    
df1 = pd.DataFrame(newIP)  #a new dataframe with the above obtained series
df1.head()
df['Src IP'] = df1  #replacing column Src IP with df1
df.head()
newIP1 = [] #splitting the Dst IP into octets,getting first two ocets
for value in df['Dst IP']:
    IP = value.split(".")
    octet1= IP[0]
    octet2= IP[1]
    
#     print(octet2)
    newIP1.append(float(octet1 + '.' + octet2))
df2 = pd.DataFrame(newIP1)
df2.head()
df['Dst IP'] = df2
df.head()
# label encoding the data : Label and Label.1
from sklearn.preprocessing import LabelEncoder 
  
Le = LabelEncoder() 
  
df['Label']= Le.fit_transform(df['Label'])
df['Label.1']= Le.fit_transform(df['Label.1'])

df.head()

df5=df.drop(['Flow ID','Timestamp'], axis = 1) #dropping the unnecessary columns
df5.head()<jupyter_output><empty_output><jupyter_text>In the next 4 steps, i will be performing some operations on the below columns to convert it from exponential values to normal float values.<jupyter_code>df5['Idle Mean']=df5['Idle Mean']/1e15
df5['Idle Max']=df5['Idle Max']/1e15
df5['Idle Min']=df5['Idle Min']/1e15
df5['Idle Std']=df5['Idle Std']/1e7
df5.head(5)
df5.isnull().values.any() 
df5.fillna(df5.mean()).head(5) #filling null values with the mean of the column
df5.notnull().values.all() #this shows there are no more null values


df5=df5.astype(float)
# Replacing infinite with nan 
df5.replace([np.inf, -np.inf], np.nan, inplace=True) 
  
# Dropping all the rows with nan values 
df5.dropna(inplace=True) 
df5.head(5)

df5['Label.1'].value_counts()<jupyter_output><empty_output><jupyter_text>So the data is highly imbalanced.....Now I will undersample class 0.0 wrt class 1.0 andoversample class 3.0 wrt class 2.0<jupyter_code>#1. Find the number of the minority class
p2p = len(df5[df5['Label.1']==7])
browsing = len(df5[df5['Label.1']==2])
audio = len(df5[df5['Label.1']==1])
chat = len(df5[df5['Label.1']==3])
file_transfer = len(df5[df5['Label.1']==5])
video = len(df5[df5['Label.1']==9])
email = len(df5[df5['Label.1']==4])
voip = len(df5[df5['Label.1']==8])

print(p2p)
print(browsing)
print(audio)
print(chat)
print(file_transfer)
print(video)
print(email)
print(voip)
p2p = df5[df5['Label.1']==7].index
browsing = df5[df5['Label.1']==2].index
audio = df5[df5['Label.1']==1].index
chat = df5[df5['Label.1']==3].index
file_transfer = df5[df5['Label.1']==5].index
video = df5[df5['Label.1']==9].index
email = df5[df5['Label.1']==4].index
voip = df5[df5['Label.1']==8].index
# load and summarize the dataset
from pandas import read_csv
from collections import Counter
from matplotlib import pyplot
from sklearn.preprocessing import LabelEncoder
data = df5.values
# split into input and output elements
X, y = data[:, :-1], data[:, -1]
# label encode the target variable
y = LabelEncoder().fit_transform(y)
# summarize distribution
counter = Counter(y)
for k,v in counter.items():
	per = v / len(y) * 100
	print('Class=%d, n=%d (%.3f%%)' % (k, v, per))
# plot the distribution
pyplot.bar(counter.keys(), counter.values())
pyplot.show()
#Forming a new dataframe for the target variable and removing it from the above dataset.
target = df5.filter(['Label.1'], axis=1)
target.head()

# example of oversampling a multi-class classification dataset
from pandas import read_csv
from imblearn.over_sampling import SMOTE
from collections import Counter
from matplotlib import pyplot
from sklearn.preprocessing import LabelEncoder
df5.drop('Label.1', inplace=True, axis=1)
data = df5.values

oversample = SMOTE()
X, target = oversample.fit_resample(X, target)
# summarize distribution
counter = Counter(target)
for k,v in counter.items():
	per = v / len(target) * 100

df5.shape
df5.columns
balance_df = pd.DataFrame(X,columns = df5.columns)
balance_df.head()
balance_df.shape
#MinMaxScaling

import pandas as pd
from sklearn import preprocessing

x = balance_df.values #returns a numpy array
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)
df_final = pd.DataFrame(x_scaled,columns = balance_df.columns)

df_final.head()
df_final.shape
#DOING THE TRAIN TEST SPLIT
from sklearn.model_selection import train_test_split
# split into train test sets
y=target
X_train, X_test, y_train, y_test = train_test_split(df_final,y, test_size=0.3)
print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
from numpy import array
from sklearn.model_selection import KFold
# data sample
data = array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6])
# prepare cross validation
kfold = KFold(3, True, 1)
# enumerate splits
for train, test in kfold.split(data):
	print('train: %s, test: %s' % (X_train, X_test))
#FIRST I AM RUNNING WITHOUT HYPERPARAMETERE TUNING|
# Load libraries
from sklearn.ensemble import AdaBoostClassifier
from sklearn import datasets
# Import train_test_split function
from sklearn.model_selection import train_test_split
#Import scikit-learn metrics module for accuracy calculation
from sklearn import metrics
# Create adaboost classifer object
abc = AdaBoostClassifier(n_estimators=50,
                         learning_rate=1)
# Train Adaboost Classifer
model = abc.fit(X_train, y_train)

#Predict the response for test dataset
y_pred = model.predict(X_test)
# Model Accuracy, how often is the classifier correct?
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
from sklearn.metrics import precision_recall_fscore_support as score


precision, recall, fscore, support = score(y_test, y_pred)

print('precision: {}'.format(precision))
print('recall: {}'.format(recall))
print('fscore: {}'.format(fscore))
print('support: {}'.format(support))
from sklearn.metrics import confusion_matrix
df_con = confusion_matrix(y_test, y_pred)
df_con
import matplotlib.pyplot as plt
def plot_confusion_matrix(df_con, title='Confusion matrix', cmap=plt.cm.gray_r):
    plt.matshow(df_con, cmap=cmap) # imshow
    #plt.title(title)
    plt.colorbar()


plot_confusion_matrix(df_con)
#NOW I NEED TO DO HYPER PARAMATER TUNING OF ADABOOST
# example of grid searching key hyperparameters for adaboost on a classification dataset
from sklearn.datasets import make_classification
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import AdaBoostClassifier
# define dataset
X, y = make_classification(n_samples=1000, n_features=20, n_informative=15, n_redundant=5, random_state=6)
# define the model with default hyperparameters
model = AdaBoostClassifier()
# define the grid of values to search
grid = dict()
grid['n_estimators'] = [10, 50, 100]
grid['learning_rate'] = [ 0.01, 0.1, 1.0]
# define the evaluation procedure
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
# define the grid search procedure
grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='accuracy')
# execute the grid search
grid_result = grid_search.fit(X_train, y_train)
# summarize the best score and configuration
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
# summarize all scores that were evaluated
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12580799616949964, 'fraction_numerical': 0.026574096241321524, 'mean_word_length': 5.662679425837321, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '29655', 'license': 'MIT', 'n_tokens_mistral': 3024, 'n_tokens_neox': 2901, 'n_words': 822, 'path': 'Review 3/[CODE] Label 2/MODEL FOR LABEL 2.ipynb', 'repo_name': 'wimpywarlord/darknet2020ML', 'size': '8354'}