<jupyter_start><jupyter_text>Plotting<jupyter_code>from pathlib import Path
import SimplePreprocessor as sp

DATASETPATH = Path("../dataset/")

pr = sp.SimplePreprocessor(deltas=True, discretize=False, flevel="MAGIK")
netdata = pr.load_path(DATASETPATH)
netdata["_date"] = netdata.index.get_level_values("_time").strftime('%a %d %b %y')
import numpy as np
import ipywidgets as widgets
from IPython.display import display, Markdown
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from ipywidgets import HBox, VBox, interactive, Layout


devices_idxs = netdata.index.droplevel(2).unique()
devices = [f"{host} ({cat})" for cat, host in devices_idxs]
devices.sort()

available_channels = [c for c in netdata.columns if (("time" not in c) and (c[0] != "_"))]
available_channels.sort()

available_days = np.unique(netdata["_date"])


# ----- ----- WIDGETS ----- ----- #
# ----- ----- ------- ----- ----- #
device_w_list = widgets.Dropdown(options=devices)
days_w_list = widgets.Dropdown(options=available_days)
selectedc_w_list = widgets.SelectMultiple(options=available_channels,
                                          description='Channel',
                                          layout=Layout(width='400px'))
timerange_slider = widgets.FloatSlider(min=.005, max=1., step=.005)
smoothing_slider = widgets.FloatSlider(min=0, max=79, step=4,
                                       description="Smoothing (aggregate x minutes)")
offset_slider = widgets.FloatSlider(min=.0, max=1., step=.01)
ts_selector = HBox([device_w_list, days_w_list])
col_selector = HBox([selectedc_w_list])
ts_shifting = HBox([timerange_slider, offset_slider])
wlist = VBox([ts_selector, col_selector, ts_shifting, smoothing_slider])


# ----- ----- PLOTTER ----- ----- #
# ----- ----- ------- ----- ----- #
def mprint(s):
    display(Markdown(s))
    
def randcolors(n):
    hexl = list('0123456789ABCDEF')
    hexc = np.random.choice(hexl, size=(n, 6))
    return ['#' + ''.join(x) for x in hexc]
    
def remove_empty(data):
    empty_cols = [ c for c in data.columns if (data[c]==0).all() ]
    for c in empty_cols:
        mprint(f"**<span style='color: red'>Empty series:</span> {c}**")
    return data.drop(empty_cols, axis=1)
    
def datetime2xaxis(dtseries, smoothing):
    if len(dtseries) <= 50:
        return "%a - %H:%M:%S"
    elif len(dtseries) <= 100:
        return "%a - %H:%M"
    else:
        return "%a - %H"
    
def describe_mtimeseries(plotname, data, smoothing=1):
    # Data description ..... #
    mprint(f"### {plotname}")
    start = min(data.index)
    end = max(data.index)
    mprint(f"**Time range**: {start} **/** {end}")
    mprint(f"**Total data range:** {end-start}")
    mprint(f"**Samples shown**: {len(data)}")
    mprint(f"**Smoothing**: {int(smoothing / 4)} minutes")

    if len(data) <= 50:
        xaxis_format = "%a - %H:%M:%S"
    elif len(data) <= 100:
        xaxis_format = "%a - %H:%M"
    else:
        xaxis_format = "%a - %H"
    
    # Plotting clean data ..... #
    empty_cols = []
    legend = []
    data = remove_empty(data)
    
    # Smoothing ..... #
    channels = data.drop(["_isanomaly"], axis=1).columns
    data[channels] = data[channels].rolling(smoothing, center=True).sum() / smoothing
    data = data.dropna()
    
    anomaly_mask = (data["_isanomaly"] != "none")
    for idx, c in enumerate(channels):
        legend.append(c)
        fig, ax = plt.subplots(figsize=(12, 6))
        ax.format_xdata = mdates.DateFormatter(xaxis_format)
        
        ax.plot(data.index, data[c])
        fig.autofmt_xdate()
        
        if anomaly_mask.any():
            attack_data = data[anomaly_mask]
            for anomalyname, anomalydata in attack_data.groupby("_isanomaly"):
                legend.append(anomalyname)
                anomalydata = anomalydata.drop("_isanomaly", axis=1)
                ax.plot(anomalydata.index, anomalydata.values)
                fig.autofmt_xdate()
                
        fig.suptitle(f"{c}", fontweight="bold")
        plt.legend(legend)
        plt.show()

        
# ----- ----- INTERACTOR ----- ----- #
# ----- ----- ---------- ----- ----- #
def whandler(device, day, channel, timerange, offset, smoothing):
    split = device.split(" ")
    host = split[0].strip()
    category = " ".join(split[1:]).replace("(", "").replace(")", "").strip()
    
    data = netdata[netdata["_date"]==day]
    chs = set(channel)
    chs.add("_isanomaly")
    chs = list(chs)
    data = data.loc[category, host][chs]
    
    # Filtering time range
    full_length = len(data)
    start_idx = int(full_length * offset)
    end_idx = min(start_idx + int(full_length * timerange), full_length)
    data = data.iloc[start_idx:end_idx]
    
    describe_mtimeseries(device, data, int(smoothing+1))

%matplotlib inline
output = widgets.interactive(whandler,
                             device=device_w_list, day=days_w_list, 
                             channel=selectedc_w_list, 
                             timerange=timerange_slider, 
                             offset=offset_slider,
                             smoothing=smoothing_slider).children[-1]
display(wlist)
display(output)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1670186216164331, 'fraction_numerical': 0.010558648492992897, 'mean_word_length': 2.54421768707483, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '11451', 'license': 'MIT', 'n_tokens_mistral': 1776, 'n_tokens_neox': 1696, 'n_words': 327, 'path': 'notebooks/DatasetVisualizationTool.ipynb', 'repo_name': 'samuelesabella/Detecting-network-anomalies-using-the-feature-space-latent-representation', 'size': '5209'}