<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from sklearn import svm
from sklearn import metrics
train_url = "https://raw.githubusercontent.com/jafetimbre/verbose-projects/master/infosec/datasets/NSL-KDD/simple/KDDTrain%2B.txt"
train20_url = "https://raw.githubusercontent.com/jafetimbre/verbose-projects/master/infosec/datasets/NSL-KDD/simple/KDDTrain%2B_20Percent.txt"

test_url = "https://raw.githubusercontent.com/jafetimbre/verbose-projects/master/infosec/datasets/NSL-KDD/simple/KDDTest%2B.txt"
test_21_url = "https://raw.githubusercontent.com/jafetimbre/verbose-projects/master/infosec/datasets/NSL-KDD/simple/KDDTest-21.txt"
nls_columns = [
    "duration",
    "protocol_type",
    "service",
    "flag",
    "src_bytes",
    "dst_bytes",
    "land",
    "wrong_fragment",
    "urgent",
    "hot",
    "num_failed_logins",
    "logged_in",
    "num_compromised",
    "root_shell",
    "su_attempted",
    "num_root",
    "num_file_creations",
    "num_shells",
    "num_access_files",
    "num_outbound_cmds",
    "is_host_login",
    "is_guest_login",
    "count",
    "srv_count",
    "serror_rate",
    "srv_serror_rate",
    "rerror_rate",
    "srv_rerror_rate",
    "same_srv_rate",
    "diff_srv_rate",
    "srv_diff_host_rate",
    "dst_host_count",
    "dst_host_srv_count",
    "dst_host_same_srv_rate",
    "dst_host_diff_srv_rate",
    "dst_host_same_src_port_rate",
    "dst_host_srv_diff_host_rate",
    "dst_host_serror_rate",
    "dst_host_srv_serror_rate",
    "dst_host_rerror_rate",
    "dst_host_srv_rerror_rate",
    "class",
    "undefined"
    ]
    
print(len(nls_columns))
df = pd.read_csv(train_url, header=None, names=nls_columns)
attack_types = {
    "normal": "normal",
    
    "back": "DoS",
    "land": "DoS",
    "neptune": "DoS",
    "pod": "DoS",
    "smurf": "DoS",
    "teardrop": "DoS",
    "mailbomb": "DoS",
    "apache2": "DoS",
    "processtable": "DoS",
    "udpstorm": "DoS",
    
    "ipsweep": "Probe",
    "nmap": "Probe",
    "portsweep": "Probe",
    "satan": "Probe",
    "mscan": "Probe",
    "saint": "Probe",

    "ftp_write": "R2L",
    "guess_passwd": "<PASSWORD>",
    "imap": "R2L",
    "multihop": "R2L",
    "phf": "R2L",
    "spy": "R2L",
    "warezclient": "R2L",
    "warezmaster": "R2L",
    "sendmail": "R2L",
    "named": "R2L",
    "snmpgetattack": "R2L",
    "snmpguess": "R2L",
    "xlock": "R2L",
    "xsnoop": "R2L",
    "worm": "R2L",
    
    "buffer_overflow": "U2R",
    "loadmodule": "U2R",
    "perl": "U2R",
    "rootkit": "U2R",
    "httptunnel": "U2R",
    "ps": "U2R",    
    "sqlattack": "U2R",
    "xterm": "U2R"
}
df["attack_type"] = df["class"].apply(lambda x: attack_types[x])
df[["class", "attack_type"]]
df["attack_type"].value_counts()
df["class"].value_counts()
SEL_ATTACK = "satan"
# drop unused columns
tmp = df.drop(["attack_type", "undefined"], axis=1)

# Filter normal and selected attack traffic 
tmp = tmp.loc[(df["class"] == "normal") | (df["class"] == SEL_ATTACK)]

# reset the index
all_data = tmp.reset_index(drop=True)

# Check ratios of the two states
all_data["class"].value_counts(normalize=True)
# Delete columns with only 0's
all_data = all_data.loc[:, (all_data != 0).any(axis=0)]
remap_dict = { 
    "class": {
        "normal": 0.0,
        "satan": 1.0
    }
}
all_data = all_data.replace(remap_dict)

non_num_cols = all_data.select_dtypes(exclude=[np.number]).columns
all_data = all_data.drop(columns=non_num_cols)
all_data.head()
# Min-max normalization
all_data = (all_data-all_data.min())/(all_data.max()-all_data.min())
all_data.head()
# prepare test data
df = pd.read_csv(test_url, header=None, names=nls_columns)
df = df.loc[:, df.columns.isin(all_data.columns)]
tmp = df.loc[(df["class"] == "normal") | (df["class"] == SEL_ATTACK)]
test_data = tmp.reset_index(drop=True)
test_data = test_data.replace(remap_dict)
test_data = (test_data-test_data.min())/(test_data.max()-test_data.min())
test_data = test_data.fillna(0.0)
test_data
inp_size = len(all_data.columns) - 1
inp_size
model = keras.Sequential([
    keras.layers.Dense(inp_size, input_dim=inp_size),
    keras.layers.Dense(units=100, activation='relu'),
    keras.layers.Dense(units=1, activation="sigmoid")
])
model.compile(optimizer='adam',
              loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              metrics=['accuracy'])
history = model.fit(all_data.iloc[:, :-1], all_data.iloc[:, [-1]], epochs=20, batch_size=200)
plt.plot(history.history['loss'])
loss, acc1 = model.evaluate(test_data.iloc[:, :-1], test_data.iloc[:, [-1]])<jupyter_output>89/327 [=======>......................] - ETA: 0s - loss: 0.1873 - accuracy: 0.9638<jupyter_text>------<jupyter_code>clf = svm.SVC(kernel='linear')

train_x = all_data.iloc[:, :-1].to_numpy()
train_y = all_data.iloc[:, [-1]].to_numpy()

test_x = test_data.iloc[:, :-1].to_numpy()
test_y = test_data.iloc[:, [-1]].to_numpy()

clf.fit(train_x, train_y)
y_pred = clf.predict(test_x)
acc2 = metrics.accuracy_score(test_y, y_pred)
print("Accuracy:", acc2)
def calc_prob(nr):
    d = 1 - np.abs(np.array([nr]*3)-np.array([.0,.5,1]))
    s = np.sum(d)
    return d / s
def comb_detector(input_vec):
    d1 = model.predict(input_vec)[0][0]
    d2 = clf.predict(input_vec.values)[0]

    d1 = calc_prob(d1)
    d2 = calc_prob(d2)
    cross = np.reshape([i*j for i in d1 for j in d2], (3,3)).T

    k = cross[0][1] + cross[1][0]
    km = 1 - k

    m_normal = (cross[0][0] + cross[0][2] + cross[2][0]) / km
    m_anom = (cross[1][1] + cross[1][2] + cross[2][1]) / km
    m_cant = cross[2][2] / km

    return np.array([m_normal, m_cant, m_anom])
def format_det(val):
    vals = np.around(val * 100, decimals=2)
    print("Detector probs:")
    print(f"\tNormal: {vals[0]} %")
    print(f"\tO: {vals[1]} %")
    print(f"\tAnomaly: {vals[2]} %")


atc = test_data.loc[test_data["class"] == 1.0].iloc[:1, :-1]

comb_det_val = comb_detector(atc)
format_det(comb_det_val)
def detector(x):
    true_val = 0
    for ix in range(len(x)):
        dX = x.iloc[ix:ix+1, :-1]
        dY = x.iloc[ix:ix+1, [-1]].values[0][0]
        comb = comb_detector(dX)
        m_comb = np.max(comb)
        if (m_comb == comb[0] and dY == 0.0) or (m_comb == comb[2] and dY == 1.0):
            true_val += 1
    return true_val / len(x)
num_det_rows = 1000
acc3 = detector(test_data[:num_det_rows])
fig, ax = plt.subplots(figsize=(10, 8))
labels = ["DNN", "SVM", "Combined model"]
vals = np.around([acc1, acc2, acc3], decimals=2)
ax.bar(labels, vals)
ax.set_title("Accuracy comparation for the 3 models")
ax.axhline(y=1,linewidth=1, color='r')
for i, v in enumerate(vals):
    ax.text(i-.07, v-.1, str(v), color='white', fontweight='bold')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19037134889182444, 'fraction_numerical': 0.02363129311610157, 'mean_word_length': 3.9234104046242773, 'pattern_counts': {'":': 43, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '663767', 'license': 'MIT', 'n_tokens_mistral': 2922, 'n_tokens_neox': 2843, 'n_words': 462, 'path': 'infosec/nsl-kdd-ids/ids.ipynb', 'repo_name': 'jafetimbre/verbose-projects', 'size': '6806'}