<jupyter_start><jupyter_text>Imports and setup<jupyter_code>import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
%config Completer.use_jedi = False<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>path_train = "../data/kddcup.data"
path_test = "../data/corrected"
path_names = "../data/kddcup.names"
# path_types = "../data/training_attack_types.txt"<jupyter_output><empty_output><jupyter_text>Load labels and datasets<jupyter_code># types_df = pd.read_csv(path_types, header=None, sep=" ")
# types_types = list(types_df[0])
# types_types.append("normal")
# print(f"len(types_types): {len(types_types)}")
names_types = []
names_names = []
with open(path_names) as f:
    for i, line in enumerate(f.readlines()):
        if i == 0:
            names_types = line.replace('.','').strip().split(',')
        else:
            names_names.append(line.replace(':','').strip().split(' ')[0])
names_names.append("label")
print(f"len(names_types): {len(names_types)}, len(names_names): {len(names_names)}")
train_df = pd.read_csv(path_train, header=None, names=names_names)
train_df.head()
test_df = pd.read_csv(path_test, header=None, names=names_names)
test_df.head()<jupyter_output><empty_output><jupyter_text>Look into data<jupyter_code>non_numerical_names = ['protocol_type', 'service', 'flag', 'label']
train_df.describe()
test_df.describe()
for name in non_numerical_names:
    unique = train_df[name].unique()
    unique.sort()
    print(f"====================================================================")
    print(f"================== {'train_df':8}:{name:>16}:{len(unique):>4} ==================")
    for i in range(0, len(unique), 4):
        print(''.join(f"{e:>16}|" for e in unique[i:i+4]))
    unique = test_df[name].unique()
    unique.sort()
    print(f"================== {'test_df':8}:{name:>16}:{len(unique):>4} ==================")
    for i in range(0, len(unique), 4):
        print(''.join(f"{e:>16}|" for e in unique[i:i+4]))
    print("\n\n")
# columns=["duration", "protocol_type", "service", "flag", "src_bytes", "dst_bytes", "land", "wrong_fragment", "urgent", "hot", "num_failed_logins", "logged_in", "num_compromised", "root_shell", "su_attempted", "num_root", "num_file_creations", "num_shells", "num_access_files", "num_outbound_cmds", "is_host_login", "is_guest_login", "count", "srv_count", "serror_rate", "srv_serror_rate", "rerror_rate", "srv_rerror_rate", "same_srv_rate", "diff_srv_rate", "srv_diff_host_rate", "dst_host_count", "dst_host_srv_count", "dst_host_same_srv_rate", "dst_host_diff_srv_rate", "dst_host_same_src_port_rate", "dst_host_srv_diff_host_rate", "dst_host_serror_rate", "dst_host_srv_serror_rate", "dst_host_rerror_rate", "dst_host_srv_rerror_rate", "label"]
# df = pd.read_csv(path, header=None)
# df.columns = columns
# df.columns
# df.head()
# df.mean()
# df.std()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22553045859000684, 'fraction_numerical': 0.007186858316221766, 'mean_word_length': 7.280453257790368, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '416253', 'license': 'MIT', 'n_tokens_mistral': 1112, 'n_tokens_neox': 1090, 'n_words': 163, 'path': 'notebooks/initial_test.ipynb', 'repo_name': 'kklgf/io_ids', 'size': '2922'}