<jupyter_start><jupyter_text><jupyter_code># Colab library to upload files to notebook
!wget https://github.com/labonisarkar/IoT/raw/master/data/mainSimulationAccessTraces.zip
!unzip mainSimulationAccessTraces.zip<jupyter_output>Archive:  mainSimulationAccessTraces.zip
replace mainSimulationAccessTraces.csv? [y]es, [n]o, [A]ll, [N]one, [r]ename: y
  inflating: mainSimulationAccessTraces.csv<jupyter_text>Importing Libraries<jupyter_code>import matplotlib.pyplot as plt # Library for Data Visulization
import pandas as pd # Library for DataFrame 
import seaborn as sns #For Data Visulization
iot = pd.read_csv('mainSimulationAccessTraces.csv')<jupyter_output><empty_output><jupyter_text>Data PreProcessing Change the name of Iot Lables in Simpler formExample: anomalous(DoSattack)  ---> DoS Attack<jupyter_code>iot['normality'].value_counts()
normality = iot['normality'].map({'normal': 'Normal', 'anomalous(DoSattack)':'DoS Attack', 'anomalous(scan)':'Scan',
       'anomalous(malitiousControl)':'Malitious Control', 'anomalous(malitiousOperation)' : 'Malitious Operation',
       'anomalous(spying)':'Spying', 'anomalous(dataProbing)':'Data Probing', 'anomalous(wrongSetUp)':'Wrong SetUp'})

iot['normality'] = normality
iot['normality'].value_counts()<jupyter_output><empty_output><jupyter_text>Visualize Missing ValuesThe Missing values are shown in block for all the column in the plot<jupyter_code>#Visualize Missing Values
plt.subplots(figsize=(20,15))
sns.heatmap(iot.isnull(), cbar=False, cmap="YlGnBu")
print(iot.isnull().sum())
iot.isnull().sum()<jupyter_output><empty_output><jupyter_text>comment- The Result shows The Columns **Values** and **accessedNodeType** have 2050 and 148 missing values- The missing values are then replaced by fill forword method for values and accessedNodeType with '/Unknown'<jupyter_code>iot['value'].fillna( method ='ffill', inplace = True) 
iot['accessedNodeType'] = iot['accessedNodeType'].fillna(value='/Unknown')

#Visualize Missing Values
plt.subplots(figsize=(20,15))
sns.heatmap(iot.isnull(), cbar=False, cmap="YlGnBu")
print(iot.isnull().sum())<jupyter_output>sourceID                     0
sourceAddress                0
sourceType                   0
sourceLocation               0
destinationServiceAddress    0
destinationServiceType       0
destinationLocation          0
accessedNodeAddress          0
accessedNodeType             0
operation                    0
value                        0
timestamp                    0
normality                    0
dtype: int64<jupyter_text>Ploting the Type of Attacks<jupyter_code>#ploting the bar graph of Types of 
plt.figure(figsize=(20,5))
attack_count = iot['normality'].value_counts()
g = sns.barplot(attack_count.index,attack_count.values,alpha=0.8)
g.set_yscale("log")
plt.title('Types of IoT Attack')
plt.xlabel('Attacks Type')
plt.ylabel('Numbers of Attack')
plt.show()

fig = plt.figure()

# Divide the figure into a 4x1 grid, and give me the first section
ax1 = fig.add_subplot(611)

# Divide the figure into a 4x1 grid, and give me the second section
ax2 = fig.add_subplot(612)

# Divide the figure into a 4x1 grid, and give me the first section
ax3 = fig.add_subplot(613)

# Divide the figure into a 4x1 grid, and give me the second section
ax4 = fig.add_subplot(614)

# Divide the figure into a 4x1 grid, and give me the first section
ax5 = fig.add_subplot(615)

# Divide the figure into a 4x1 grid, and give me the second section
ax6 = fig.add_subplot(616)


iot.groupby(['normality','sourceType'])['value'].count().drop(['Normal']).unstack().plot(kind='barh',stacked=True,figsize=(20,20),ax=ax1)
iot.groupby(['normality','sourceLocation'])['value'].count().drop(['Normal']).unstack().plot(kind='barh',stacked=True,figsize=(20,20),ax=ax2)
iot.groupby(['normality','destinationServiceType'])['value'].count().drop(['Normal']).unstack().plot(kind='barh',stacked=True,figsize=(20,20),ax=ax3)
iot.groupby(['normality','destinationLocation'])['value'].count().drop(['Normal']).unstack().plot(kind='barh',stacked=True,figsize=(20,20),ax=ax4)
iot.groupby(['normality','operation'])['value'].count().drop(['Normal']).unstack().plot(kind='barh',stacked=True,figsize=(20,20),ax=ax5)
iot.groupby(['normality','accessedNodeType'])['value'].count().drop(['Normal']).unstack().plot(kind='barh',stacked=True,figsize=(20,20),ax=ax6)
ax1.legend(ncol=2,title='Source Type')
ax2.legend(ncol=4,title='Source Location')
ax3.legend(ncol=2,title='Destination Type')
ax4.legend(ncol=4,title='Destination Location')
ax5.legend(title='Operation')
ax6.legend(ncol=2,title='Accessed Node Type')<jupyter_output><empty_output><jupyter_text>Data Pre Processing- The Value Columns show some anomalies in the data, The next steps we cleans the valus data and replace- Example : none to 0 , false to 0 , true to 1 , twenty to 20- Next we change the data type with string to numaric<jupyter_code>iot[['value']][iot['value'].str.isalnum()]['value'].value_counts()
#Map the Non-Numeric Value to Corresponding values
value_map = {'none':'0','false':'0','true':'1','twenty':'20'}
iot['value'] = iot['value'].map(value_map).fillna(iot['value'])
# covert the data values to numeric
iot['value'] = pd.to_numeric(iot['value'], errors='coerce')
print(iot.isnull().sum())
iot.dropna(inplace=True)
print(iot.isnull().sum())<jupyter_output>sourceID                     0
sourceAddress                0
sourceType                   0
sourceLocation               0
destinationServiceAddress    0
destinationServiceType       0
destinationLocation          0
accessedNodeAddress          0
accessedNodeType             0
operation                    0
value                        0
timestamp                    0
normality                    0
dtype: int64<jupyter_text>Feature Genaration<jupyter_code>#Conver columns to type to category
def to_category(dataframe,columns=[],type = 'None'):
    if type == 'object':
        columns = list(dataframe.select_dtypes(type).dtypes.index)        
    return dataframe[columns].apply(lambda x: x.astype('category'))

# Function OneHot Coding:
def convert_to_onehot(dataframe,feature_list,drop_first=True):
    """Covert all columns in the list using One Hot Method"""
    res = pd.DataFrame(index=dataframe.index)
    for feature in feature_list:
        dummies = pd.get_dummies(dataframe[[feature]])
        res = pd.concat([res,dummies],axis=1)
    if not drop_first:
        res = pd.concat([dataframe,res],axis=1)
    return res


#label Encoder for categorical data
def label_encoder(dataframe,columns,drop_first=True,prefix='_label'):
    """convert categorical data with label encoder"""
    from sklearn.preprocessing import LabelEncoder
    res = pd.DataFrame(index=dataframe.index)
    for col in columns:
        # creating instance of labelencoder
        labelencoder = LabelEncoder()
        res[col+prefix] = labelencoder.fit_transform(dataframe[col])
    if not drop_first:
        res = pd.concat([dataframe,res],axis=1)
    return res
iot.dtypes
columns = to_category(iot,type = 'object').columns
iot[columns] = to_category(iot,type = 'object')
iot.dtypes
iot.describe()
iot.select_dtypes('category').describe()
#unique_values = iot.select_dtypes('category').describe().loc['unique',:].to_frame()
#label_col = unique_values[unique_values > 21].dropna().index
#hot_col = unique_values[unique_values <= 21].dropna().index

iot_feature_col = ['sourceType', 'sourceLocation', 'destinationServiceType', 'destinationLocation', 'accessedNodeType', 'operation','sourceID', 'sourceAddress', 'destinationServiceAddress', 'accessedNodeAddress']
target = ['normality']
iot.drop('timestamp',axis=1,inplace=True)
iot_feature = pd.concat([label_encoder(iot,iot_feature_col),iot[['value']]],axis=1)
iot[['normality']]
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
X = scaler.fit_transform(iot_feature)

from sklearn.preprocessing import LabelEncoder
labelencoder_y = LabelEncoder()
y = labelencoder_y.fit_transform(iot['normality'].values).reshape(-1,1)
attack_class = list(labelencoder_y.classes_)
list(labelencoder_y.inverse_transform([0, 1, 2, 3, 4, 5, 6, 7]))
from sklearn.preprocessing import label_binarize
y = label_binarize(y, classes=[0, 1, 2, 3, 4, 5, 6, 7])
y[1]<jupyter_output><empty_output><jupyter_text>Machine Learning<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
from sklearn.metrics import log_loss
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score,recall_score,precision_score,f1_score
from sklearn.multiclass import OneVsRestClassifier
#Train Test Split
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state = 123)
# ploting Auc Ruc Curve
def add_results(name,y_test,y_pred):
  results[name] = {}
  results[name]['Accuracy Score'] = round(accuracy_score(y_test,y_pred),4)
  results[name]['Precision Score'] = round(precision_score(y_test,y_pred,average='weighted'),4)
  results[name]['Recall Score'] = round(recall_score(y_test,y_pred,average='weighted'),4)
  results[name]['F1 Score'] = round(f1_score(y_test,y_pred,average='weighted'),4)
  print(results[name])
  return results
#ROC AUC plot

from sklearn.metrics import roc_curve, auc
from scipy import interp
from itertools import cycle

def plot_roc_curve(y_score,y_test,n_classes = 8,name="RoC Curve"):
  fpr = dict()
  tpr = dict()
  roc_auc = dict()
  attack_class = ['D. Prob', 'DoS', 'Mal. Cont.','Mal. Op.', 'Normal', 'Scan', 'Spying', 'W. SetUp']
  for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])
  #print(roc_auc)
  #plot
  classes = attack_class
  plt.figure(figsize=(7,6))
  ax=plt.subplot(111)
  colors = cycle(['red','maroon','navy','aqua', 'darkorange', 'cornflowerblue','olive','darkslategray'])
  for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=2,
             label='{0} (area = {1:0.2f})'
             ''.format(classes[i], roc_auc[i]))
  plt.plot([0, 1], [0, 1], 'k--', lw=lw)
  plt.xlim([0.0, 1.0])
  plt.ylim([0.0, 1.0])
  plt.xticks(fontsize=11,weight='bold')
  plt.yticks(fontsize=11,weight='bold')
  plt.setp(ax.spines.values(), linewidth=2)
  ax.xaxis.set_tick_params(width=2)
  ax.yaxis.set_tick_params(width=2)
  plt.xlabel('False Positive Rate',fontsize=12,weight='bold')
  plt.ylabel('True Positive Rate',fontsize=12,weight='bold')
  plt.legend(loc="lower right")
  plt.savefig(name)

from sklearn.metrics import classification_report
# All Machine Learning Code Results
results = {}<jupyter_output><empty_output><jupyter_text>K Nearest Neighbor(KNN)<jupyter_code>name = "KNeighborsClassifier"

from sklearn.neighbors import KNeighborsClassifier
knn = OneVsRestClassifier(KNeighborsClassifier(n_neighbors= 8))

knn.fit(X_train,y_train)
y_pred  = knn.predict(X_test)

add_results(name,y_test,y_pred)
print(classification_report(y_test, y_pred,target_names=attack_class ))
y_score = knn.predict_proba(X_test)
plot_roc_curve(y_score,y_test)
from sklearn.metrics import roc_curve, auc
fpr_svc = dict()
tpr_svc = dict()
roc_auc_svc = dict()
for i in range(8):
    fpr_svc[i], tpr_svc[i], _ = roc_curve(y_test[:, i], y_score[:, i])
    roc_auc_svc[i] = auc(fpr_svc[i], tpr_svc[i])
y_score[:, 1]
y_score = knn.predict_proba(X_test)
from scipy import interp
from itertools import cycle
n_classes = 8
lw = 2
classes = attack_class
plt.figure(figsize=(7,6))
ax=plt.subplot(111)
colors = cycle(['red','maroon','navy','aqua', 'darkorange', 'cornflowerblue','olive','darkslategray'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr_svc[i], tpr_svc[i], color=color, lw=lw,
             label='{0} (area = {1:0.2f})'
             ''.format(classes[i], roc_auc_svc[i]))

plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xticks(fontsize=11,weight='bold')
plt.yticks(fontsize=11,weight='bold')
plt.setp(ax.spines.values(), linewidth=2)
ax.xaxis.set_tick_params(width=2)
ax.yaxis.set_tick_params(width=2)
plt.xlabel('False Positive Rate',fontsize=12,weight='bold')
plt.ylabel('True Positive Rate',fontsize=12,weight='bold')
plt.legend(loc="lower right")
plt.savefig(name)<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>from sklearn.tree import DecisionTreeClassifier
tree = DecisionTreeClassifier(criterion="entropy", max_depth = 10)
tree.fit(X_train,y_train)
y_pred  = tree.predict(X_test)

add_results("DecisionTreeClassifier",y_test,y_pred)
print(classification_report(y_test, y_pred,target_names=attack_class ))<jupyter_output>precision    recall  f1-score   support

       Data Probing       1.00      1.00      1.00        67
         DoS Attack       0.97      0.65      0.78      1121
  Malitious Control       1.00      1.00      1.00       187
Malitious Operation       1.00      1.00      1.00       150
             Normal       0.99      1.00      1.00     69630
               Scan       1.00      1.00      1.00       321
             Spying       0.95      0.64      0.76        91
        Wrong SetUp       1.00      1.00      1.00        22

          micro avg       0.99      0.99      0.99     71589
          macro avg       0.99      0.91      0.94     71589
       weighted avg       0.99      0.99      0.99     71589
        samples avg       0.99      0.99      0.99     71589<jupyter_text>Support Vector Machine<jupyter_code>from sklearn.ensemble import RandomForestClassifier
rfc = RandomForestClassifier(warm_start=True,n_estimators=10)
rfc.fit(X_train, y_train) 
y_pred = rfc.predict(X_test)

print('Accuracy Score : ' + str(accuracy_score(y_test,y_pred)))
print('Precision Score : ' + str(precision_score(y_test,y_pred,average='weighted')))
print('Recall Score : ' + str(recall_score(y_test,y_pred,average='weighted')))
print('F1 Score : ' + str(f1_score(y_test,y_pred,average='weighted')))<jupyter_output>Accuracy Score : 0.9940865872624158
Precision Score : 0.993991705237306
Recall Score : 0.9940865872624158
F1 Score : 0.9934829055703668<jupyter_text>SVM : Linear SVC<jupyter_code>from sklearn.svm import LinearSVC
lsvc = OneVsRestClassifier(LinearSVC())

lsvc.fit(X_train, y_train)
y_pred_lsvc = lsvc.predict(X_test)

print('Accuracy Score : ' + str(accuracy_score(y_test,y_pred_lsvc)))
print('Precision Score : ' + str(precision_score(y_test,y_pred_lsvc,average='weighted')))
print('Recall Score : ' + str(recall_score(y_test,y_pred_lsvc,average='weighted')))
print('F1 Score : ' + str(f1_score(y_test,y_pred_lsvc,average='weighted')))
y_score_lsvc = lsvc.decision_function(X_test)
plot_roc_curve(y_score_lsvc,y_test)
from sklearn.metrics import roc_curve, auc
fpr_svc = dict()
tpr_svc = dict()
roc_auc_svc = dict()
for i in range(8):
    fpr_svc[i], tpr_svc[i], _ = roc_curve(y_test[:, i], y_score_svc[:, i])
    roc_auc_svc[i] = auc(fpr_svc[i], tpr_svc[i])


from scipy import interp
from itertools import cycle
n_classes = 8
lw = 2
classes = attack_class
plt.figure(figsize=(7,6))
ax=plt.subplot(111)
colors = cycle(['red','maroon','navy','aqua', 'darkorange', 'cornflowerblue','olive','darkslategray'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr_svc[i], tpr_svc[i], color=color, lw=lw,
             label='{0} (area = {1:0.2f})'
             ''.format(classes[i], roc_auc_svc[i]))

plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xticks(fontsize=11,weight='bold')
plt.yticks(fontsize=11,weight='bold')
plt.setp(ax.spines.values(), linewidth=2)
ax.xaxis.set_tick_params(width=2)
ax.yaxis.set_tick_params(width=2)
plt.xlabel('False Positive Rate',fontsize=12,weight='bold')
plt.ylabel('True Positive Rate',fontsize=12,weight='bold')
plt.legend(loc="lower right")
plt.savefig("ROC_of_SVM")
roc_auc_svc<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14374960560358427, 'fraction_numerical': 0.03230895437622263, 'mean_word_length': 5.032736962314427, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 50, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '91435', 'license': 'MIT', 'n_tokens_mistral': 6010, 'n_tokens_neox': 5605, 'n_words': 1078, 'path': 'IoTNoteBook_Ver2.ipynb', 'repo_name': 'labonisarkar/IoT', 'size': '15847'}