<jupyter_start><jupyter_text>--- Data Analysis<jupyter_code>df = pd.read_csv('./Train_Test_Network_dataset/NF-ToN-IoT-v2.csv.gz')
df['Attack'].unique()
# create a dataframe with just the instances of benign and scanning traffic
df_scan = df[df['Attack'].isin(['Benign', 'scanning'])]
df_scan['Attack'].unique()
del df
df_scan.drop('Attack', axis=1, inplace=True)
# Attack (1), Benign (0)
df_scan['Label'].value_counts() # this dataset contains ALL benign plus those with Scanning
# according to paper these are the source IPs from Scanning attacks (Ground-truth)
scan_src_ip = ['192.168.1.30', '192.168.1.31', '192.168.1.32', '192.168.1.33', '192.168.1.38']
# Who are the targets of Scanning machines ?
# top targets of Scan, greater than 3k flow samples
top = df_scan[df_scan['IPV4_SRC_ADDR'].isin(scan_src_ip)]['IPV4_DST_ADDR'].value_counts().head(229).index
# which machines are targeting the Scanning machines ? (counter direction of flow)
#df_scan[df_scan['IPV4_DST_ADDR'].isin(scan_src_ip)]['IPV4_SRC_ADDR'].value_counts()
# filter dataframe for top Scanning IP destinations (considering Scanning traffic)
df_top = df_scan[df_scan['IPV4_DST_ADDR'].isin(top)]
# check the balance between scan and normal for top scan targets
# 0 is benign and 1 is attack (just Scanning)
target_balance = df_top[['IPV4_DST_ADDR', 'Label']]
target_balance = pd.get_dummies(target_balance, columns=['Label'])
#target_balance.groupby('IPV4_DST_ADDR').sum()<jupyter_output><empty_output><jupyter_text>Choose the feasible agents according to the amount of benign traffic (dst addr with both normal and scanning flow samples)<jupyter_code>feasible_agents = ['192.168.1.1', '192.168.1.133', '192.168.1.152', '192.168.1.169', '192.168.1.180', '192.168.1.186', 
       '192.168.1.190', '192.168.1.193', '192.168.1.194', '192.168.1.195', '192.168.1.46', '192.168.1.49', '192.168.1.79']
df_top = df_top[df_top['IPV4_DST_ADDR'].isin(feasible_agents)]<jupyter_output><empty_output><jupyter_text>Check the balance between scanning and normal for top scanning targets<jupyter_code>target_balance = df_top[['IPV4_DST_ADDR', 'Label']]
target_balance = pd.get_dummies(target_balance, columns=['Label'])
# Attack (1), Benign (0)
target_balance.groupby('IPV4_DST_ADDR').sum()
remove_columns = ['IPV4_SRC_ADDR', 'L4_SRC_PORT', 'IPV4_DST_ADDR', 'L4_DST_PORT', 'PROTOCOL', 'L7_PROTO']
df_hist = df_top.loc[:, ~df_top.columns.isin(remove_columns)]
df_hist.describe()<jupyter_output><empty_output><jupyter_text>--- Single repository with the data from all agents<jupyter_code>df_top['IPV4_DST_ADDR'].unique()
remove_columns = ['IPV4_SRC_ADDR', 'L4_SRC_PORT', 'IPV4_DST_ADDR', 'L4_DST_PORT', 'PROTOCOL', 'L7_PROTO']
df_top.drop(remove_columns, axis=1, inplace=True)
df_top.head()<jupyter_output><empty_output><jupyter_text>--- Training generic Logistic Regression model on full dataset as Baseline<jupyter_code>def preprocess(X):
    cols = X.select_dtypes(include=[np.float64]).columns
    X[cols] = X[cols].astype(np.float32)

    X.fillna(0, inplace=True)
    X.replace(np.inf, 9999, inplace=True)
    
    return X
from sklearn.linear_model import SGDClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold, GridSearchCV, train_test_split
from sklearn.metrics import balanced_accuracy_score, make_scorer, f1_score, roc_auc_score, precision_score, recall_score, roc_curve

X = df_top.drop(columns = ["Label"])
y = df_top.Label

X = preprocess(X)
    
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=seed, stratify=y) #stratified

clf = SGDClassifier(random_state=seed, loss="log", class_weight="balanced")
parameters = {}
    
prep = StandardScaler() 
X_train = prep.fit_transform(X_train)
clf.fit(X_train, y_train)
    
X_test = prep.transform(X_test) 
y_pred = clf.predict(X_test)
y_proba = clf.predict_proba(X_test)[:,1] # for ROC curve, get just for positive class "1"
    
score = f1_score(y_test, y_pred, average="binary") # binary only report results for the class 1 (pos_label)
        
print(set(y_test), set(y_pred), "{:.2f}".format(score))

results = {}
results["f1-score"] = score
results["accuracy"] = balanced_accuracy_score(y_test, y_pred)
results["precision"] = precision_score(y_test, y_pred, average="binary")
results["recall"] =  recall_score(y_test, y_pred, average="binary")
results["auc"] = roc_auc_score(y_test, y_pred)
results["fpr"], results["tpr"], results["thresholds"] = roc_curve(y_test, y_proba)
results["%benign"] = float("{:.3f}".format(sum(y==0)/len(y)))
results["%scan"] = float("{:.3f}".format(sum(y==1)/len(y)))
results["total_samples"] = len(y)
plt.style.use('seaborn')
plt.figure()

fpr, tpr, thresholds = results["fpr"], results["tpr"], results["thresholds"]
AUC = results["auc"]
    
plt.plot(fpr, tpr, label="{} (AUC={:.2f})".format("Baseline", AUC))

plt.plot([0,1], [0,1], color='gray', linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")

results<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1615187881172536, 'fraction_numerical': 0.04721621089907535, 'mean_word_length': 6.72644376899696, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 18, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '158321', 'license': 'MIT', 'n_tokens_mistral': 2078, 'n_tokens_neox': 1890, 'n_words': 376, 'path': 'Centralized.ipynb', 'repo_name': 'c2dc/wain2021', 'size': '5083'}