<jupyter_start><jupyter_text>Imports and setup<jupyter_code>import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
%config Completer.use_jedi = False<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>path_data = "../data/kddcup.data"
path_names = "../data/kddcup.names"<jupyter_output><empty_output><jupyter_text>Load labels and datasets<jupyter_code>names_types = []
names_names = []
with open(path_names) as f:
    for i, line in enumerate(f.readlines()):
        if i == 0:
            names_types = line.replace('.','').strip().split(',')
        else:
            names_names.append(line.replace(':','').strip().split(' ')[0])
names_names.append("label")
print(f"len(names_types): {len(names_types)}, len(names_names): {len(names_names)}")
whole_df = pd.read_csv(path_data, header=None, names=names_names)
whole_df.head()
whole_df.describe()
whole_df.label.hist(figsize=(30,12))
plt.yscale('log')
plt.show()<jupyter_output><empty_output><jupyter_text>Look into data<jupyter_code>non_numerical_names = ['protocol_type', 'service', 'flag', 'label']<jupyter_output><empty_output><jupyter_text>Prepare data<jupyter_code>from sklearn.preprocessing import OneHotEncoder

ll = pd.DataFrame(whole_df.label)
out = {}
BB = []
def f(x):
    a, b = pd.factorize(x)
    out[x.name] = len(b)
    BB = b
    return a
b = ll.apply(f)

encoded_df = b

for col in whole_df.columns:
    if col == 'label':
        pass
    elif col in ['protocol_type', 'service', 'flag']:
        encoded_df = pd.concat([encoded_df, pd.get_dummies(whole_df[col], prefix=col)], axis=1)
    else:
        encoded_df = pd.concat([encoded_df, whole_df[col]], axis=1)
encoded_df.head()
from sklearn.model_selection import train_test_split

train_df, test_df = train_test_split(encoded_df, test_size=0.3)
train_df.head()
import numpy as np
import pandas as pd
import tensorflow as tf

training_dataset = tf.data.Dataset.from_tensor_slices(
    (
        tf.cast(train_df[encoded_df.columns[1:]].values, tf.float32),
        tf.cast(train_df['label'].values, tf.int32)
    )
)
model = tf.keras.Sequential([
  tf.keras.layers.Flatten(),
  tf.keras.layers.Dense(100),
  tf.keras.layers.Dense(out['label'])
])

model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), 
              metrics=['accuracy'])
fmnist_train_ds = training_dataset.batch(32)
model.fit(fmnist_train_ds, epochs=10)
testing_dataset = tf.data.Dataset.from_tensor_slices(
    (
        tf.cast(test_df[encoded_df.columns[1:]].values, tf.float32),
        tf.cast(test_df['label'].values, tf.int32)
    )
)
testing_dataset_b = testing_dataset.batch(32)
results = model.evaluate(testing_dataset_b)
print("test loss, test acc:", results)
from sklearn.metrics import plot_confusion_matrix
X_test = tf.cast(train_df[encoded_df.columns[1:]].values, tf.float32),
y_test = tf.cast(train_df['label'].values, tf.int32)
preds = model.predict(X_test)
y_pred = np.argmax(preds, axis=1)
cfm = tf.math.confusion_matrix(y_test, y_pred)
import matplotlib.pyplot as plt
for row in cfm:
    print('|'.join([f'{n:6}' for n in row]))
cfm_cp = np.copy(cfm)
for i, row in enumerate(cfm):
    cfm_cp[i] = cfm_cp[i] * 255 / np.sum(row)
plt.imshow(cfm_cp)
model2 = tf.keras.Sequential([
  tf.keras.layers.Flatten(),
  tf.keras.layers.Dense(128),
  tf.keras.layers.Dropout(0.1),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(64),
  tf.keras.layers.Dropout(0.1),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(32),
  tf.keras.layers.Dropout(0.1),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(16),
  tf.keras.layers.Dropout(0.1),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(32),
  tf.keras.layers.Dense(out['label'])
])

model2.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), 
              metrics=['accuracy'])
history2 = model2.fit(fmnist_train_ds, epochs=10)
results2 = model2.evaluate(testing_dataset_b)
print("test loss, test acc:", results2)
preds = model2.predict(X_test)
y_pred = np.argmax(preds, axis=1)
cfm = tf.math.confusion_matrix(y_test, y_pred)
cfm_cp = np.copy(cfm)
for i, row in enumerate(cfm):
    cfm_cp[i] = cfm_cp[i] * 255 / np.sum(row)
plt.imshow(cfm_cp)
# columns=["duration", "protocol_type", "service", "flag", "src_bytes", "dst_bytes", "land", "wrong_fragment", "urgent", "hot", "num_failed_logins", "logged_in", "num_compromised", "root_shell", "su_attempted", "num_root", "num_file_creations", "num_shells", "num_access_files", "num_outbound_cmds", "is_host_login", "is_guest_login", "count", "srv_count", "serror_rate", "srv_serror_rate", "rerror_rate", "srv_rerror_rate", "same_srv_rate", "diff_srv_rate", "srv_diff_host_rate", "dst_host_count", "dst_host_srv_count", "dst_host_same_srv_rate", "dst_host_diff_srv_rate", "dst_host_same_src_port_rate", "dst_host_srv_diff_host_rate", "dst_host_serror_rate", "dst_host_srv_serror_rate", "dst_host_rerror_rate", "dst_host_srv_rerror_rate", "label"]
# df = pd.read_csv(path, header=None)
# df.columns = columns
# df.columns
# df.head()
# df.mean()
# df.std()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16095693779904308, 'fraction_numerical': 0.013779904306220095, 'mean_word_length': 5.52434456928839, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '236467', 'license': 'MIT', 'n_tokens_mistral': 2095, 'n_tokens_neox': 2042, 'n_words': 303, 'path': 'notebooks/neural_network_training.ipynb', 'repo_name': 'kklgf/io_ids', 'size': '5225'}