<jupyter_start><jupyter_text>Proyecto 1 - <NAME> - 16387<jupyter_code>import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
# load the datasets
df1 = pd.read_csv('dataset-part1.csv')
df2 = pd.read_csv('dataset-part2.csv')<jupyter_output><empty_output><jupyter_text>exploratory analysis<jupyter_code># first view
df1.head()
# first view
df2.head()
# merge the dataframes in order to work only with one
frames = [df1, df2]
df = pd.concat(frames, ignore_index=True)
df
# summary of the variables in the df
df.info()
# check for null entries
df.isnull().sum()
# check for NaN entries
df.isna().sum()<jupyter_output><empty_output><jupyter_text>pre-processing<jupyter_code># non numerical variables to deal with:
# 1. 'DST_TO_SRC_SECOND_BYTES'
# 2. 'IPV4_DST_ADDR'
# 3. 'IPV4_SRC_ADDR'
# 4. 'PROTOCOL_MAP'
# 5. 'SRC_TO_DST_SECOND_BYTES'
# 6. 'L7_PROTO_NAME'
# 7. 'LABEL'
# 1. 'DST_TO_SRC_SECOND_BYTES'

df['DST_TO_SRC_SECOND_BYTES'].describe()

# 569243 unique values, too many to encode, dropping it
df.drop('DST_TO_SRC_SECOND_BYTES', axis=1, inplace=True)
df.head(5)
# 2. 'IPV4_DST_ADDR'

df['IPV4_DST_ADDR'].describe()

# 440887 unique values, too many to encode, dropping it
df.drop('IPV4_DST_ADDR', axis=1, inplace=True)
df.head(5)
# 3. 'IPV4_SRC_ADDR'

df['IPV4_SRC_ADDR'].describe()

# 440887 unique values, too many to encode, dropping it
df.drop('IPV4_SRC_ADDR', axis=1, inplace=True)
df.head(5)
# 4. 'PROTOCOL_MAP'

df['PROTOCOL_MAP'].describe()

# 5 unique values, encoding it
# encoding 'PROTOCOL_MAP' to numerical values
df['PROTOCOL_MAP'].value_counts()

# tcp = 1
# udp = 2
# icmp = 3
# ipv6-icmp = 4
# gre = 5
df['PROTOCOL_MAP'] = df['PROTOCOL_MAP'].replace(to_replace='tcp', value=1)
df['PROTOCOL_MAP'] = df['PROTOCOL_MAP'].replace(to_replace='udp', value=2)
df['PROTOCOL_MAP'] = df['PROTOCOL_MAP'].replace(to_replace='icmp', value=3)
df['PROTOCOL_MAP'] = df['PROTOCOL_MAP'].replace(to_replace='ipv6-icmp', value=4)
df['PROTOCOL_MAP'] = df['PROTOCOL_MAP'].replace(to_replace='gre', value=5)

df['PROTOCOL_MAP'].value_counts()
# successfuly encoded!
# 5. 'SRC_TO_DST_SECOND_BYTES'

df['SRC_TO_DST_SECOND_BYTES'].describe()

# 3160289 unique values, too many to encode, dropping it

df.drop('SRC_TO_DST_SECOND_BYTES', axis=1, inplace=True)
df.head(5)
# 5. 'L7_PROTO_NAME'

df['L7_PROTO_NAME'].describe(), df['L7_PROTO_NAME'].value_counts()
df_test = df[['L7_PROTO_NAME', 'FIREWALL_EVENT']]
labels, unique = pd.factorize(df_test['L7_PROTO_NAME'])
df_test2 = pd.DataFrame(labels, columns = ['new_L7_PROTO_NAME'])
df_test2
# Substituting with numerical value (457 uniques)
df['L7_PROTO_NAME'] = df_test2['new_L7_PROTO_NAME']
df
# 6. 'LABEL'

df['LABEL'].value_counts()

# **TARGET VALUE**

# 4 unique values, encoding them:

# Normal flow                    = 1
# SYN Scan - aggressive          = 2
# Denial of Service R-U-Dead-Yet = 3
# Denial of Service Slowloris    = 4
df['LABEL'] = df['LABEL'].replace(to_replace='Normal flow', value=1)
df['LABEL'] = df['LABEL'].replace(to_replace='SYN Scan - aggressive', value=2)
df['LABEL'] = df['LABEL'].replace(to_replace='Denial of Service R-U-Dead-Yet', value=3)
df['LABEL'] = df['LABEL'].replace(to_replace='Denial of Service Slowloris', value=4)

df['LABEL'].value_counts()
# last checkup of the data
df.info()

# 46 numerical varibales variables 
# ready to go to selection<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 12207873 entries, 0 to 12207872
Data columns (total 46 columns):
 #   Column                      Dtype
---  ------                      -----
 0   BIFLOW_DIRECTION            int64
 1   DIRECTION                   int64
 2   FIREWALL_EVENT              int64
 3   FIRST_SWITCHED              int64
 4   FLOW_ACTIVE_TIMEOUT         int64
 5   FLOW_DURATION_MICROSECONDS  int64
 6   FLOW_DURATION_MILLISECONDS  int64
 7   FLOW_END_MILLISECONDS       int64
 8   FLOW_END_SEC                int64
 9   FLOW_ID                     int64
 10  FLOW_INACTIVE_TIMEOUT       int64
 11  FLOW_START_MILLISECONDS     int64
 12  FLOW_START_SEC              int64
 13  FRAME_LENGTH                int64
 14  IN_BYTES                    int64
 15  IN_PKTS                     int64
 16  L4_DST_PORT                 int64
 17  L4_SRC_PORT                 int64
 18  LAST_SWITCHED               int64
 19  MAX_IP_PKT_LEN              int64
 20  MIN_IP_PKT_LEN        [...]<jupyter_text>feature selection<jupyter_code># Pearson Correlation
#plt.figure(figsize=(25,25))
cor = df.corr()
#sns.heatmap(cor, annot=True, cmap=plt.cm.Reds)
#plt.show()
# correlation with output variable
cor_target = abs(cor['LABEL'])

# selecting highly correlated features
relevant_features = cor_target[cor_target>0.45]
relevant_features
highCorrDf = df[['FLOW_DURATION_MICROSECONDS', 'FLOW_DURATION_MILLISECONDS', 'PROTOCOL_MAP','TCP_FLAGS',
                 'TCP_WIN_SCALE_OUT']]
highCorrDf
target = df['LABEL']
target<jupyter_output><empty_output><jupyter_text>train/test splitting<jupyter_code># we will separate the data in train (55%), validation (15%) and test (30%) = 100%
from sklearn import metrics, model_selection, tree
# first split (train 55% and test 45%)
x_train, x_val_test, y_train, y_val_test = model_selection.train_test_split(highCorrDf, target, test_size = 0.45, random_state=321)

# second split: 33% and 67%. 
x_val, x_test, y_val, y_test = model_selection.train_test_split(x_val_test, y_val_test, test_size = 0.67, random_state=123)<jupyter_output><empty_output><jupyter_text>implementation no. 1: decision tree classifier<jupyter_code>clf = tree.DecisionTreeClassifier()
clf = clf.fit(x_train,y_train)

target_pred = clf.predict(x_val)<jupyter_output><empty_output><jupyter_text>metrics for implementation no. 1 (validation set: 15% of the data)<jupyter_code>print('validation data:')
print()
h = metrics.accuracy_score(y_val, target_pred)
print('accuracy: %0.2f' % (h*100), '%')
print('-------------------------------------------------------')
print('confusion matrix \n',metrics.confusion_matrix(y_val, target_pred))
print('-------------------------------------------------------')
print(metrics.classification_report(y_val, target_pred, target_names=['Normal Flow', 'SYN', 'RUDY', 'Slowloris']))
print('-------------------------------------------------------')<jupyter_output>validation data:

accuracy: 91.73 %
-------------------------------------------------------
confusion matrix 
 [[849305 109274   9414   8228]
 [  3437 367099     13      3]
 [  8395     23 326344   2867]
 [  5401      8   2784 120274]]
-------------------------------------------------------
              precision    recall  f1-score   support

 Normal Flow       0.98      0.87      0.92    976221
         SYN       0.77      0.99      0.87    370552
        RUDY       0.96      0.97      0.97    337629
   Slowloris       0.92      0.94      0.93    128467

    accuracy                           0.92   1812869
   macro avg       0.91      0.94      0.92   1812869
weighted avg       0.93      0.92      0.92   1812869

-------------------------------------------------------<jupyter_text>metrics for implementation no. 1 (test set: 30% of the data)<jupyter_code>target_pred_test = clf.predict(x_test)
print('test data:')
print('-------------------------------------------------------')
h = metrics.accuracy_score(y_test, target_pred_test)
print('accuracy: %0.2f' % (h*100), '%')
print('-------------------------------------------------------')
print('confusion matrix \n',metrics.confusion_matrix(y_test, target_pred_test))
print('-------------------------------------------------------')
print(metrics.classification_report(y_test, target_pred_test, target_names=['Normal Flow', 'SYN', 'RUDY', 'Slowloris']))
print('-------------------------------------------------------')
fpr, tpr, thresholds = metrics.roc_curve(y_test, target_pred_test, pos_label=2)
metrics.auc(fpr, tpr)
plt.plot(fpr, tpr)<jupyter_output><empty_output><jupyter_text>implementation no. 2: KNN classifier<jupyter_code># scaler
from sklearn.preprocessing import StandardScaler

#sc = StandardScaler()
#X_train = sc.fit_transform(x_train)
#X_test = sc.transform(x_val)

# knn
from sklearn.neighbors import KNeighborsClassifier
classifier = KNeighborsClassifier(n_neighbors = 7)
classifier.fit(x_train, y_train)

y_pred = classifier.predict(x_val)
ac = metrics.accuracy_score(y_val,y_pred)
print('validation data: ')
print('-------------------------------------------------------')
print('accuracy: %0.2f' % (100*ac),'%')
print('-------------------------------------------------------')
print('confusion matrix \n',metrics.confusion_matrix(y_val, y_pred))
print('-------------------------------------------------------')
print(metrics.classification_report(y_val, y_pred, target_names=['Normal Flow', 'SYN', 'RUDY', 'Slowloris']))
print('-------------------------------------------------------')
y_pred_test = classifier.predict(x_test)
ac = metrics.accuracy_score(y_test,y_pred_test)
print('test data: ')
print('-------------------------------------------------------')
print('accuracy: %0.2f' % (100*ac),'%')
print('-------------------------------------------------------')
print('confusion matrix \n',metrics.confusion_matrix(y_test, y_pred_test))
print('-------------------------------------------------------')
print(metrics.classification_report(y_test, y_pred_test, target_names=['Normal Flow', 'SYN', 'RUDY', 'Slowloris']))
print('-------------------------------------------------------')
cm = metrics.confusion_matrix(y_test,y_pred_test)
cm
fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred_test, pos_label=2)
metrics.auc(fpr, tpr)
plt.plot(fpr,tpr)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22008877877567876, 'fraction_numerical': 0.04531846804996387, 'mean_word_length': 2.713300114986585, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '624619', 'license': 'MIT', 'n_tokens_mistral': 3590, 'n_tokens_neox': 3304, 'n_words': 677, 'path': 'proyecto1.ipynb', 'repo_name': 'sergiomarchena16/Proyecto_1', 'size': '9703'}