<jupyter_start><jupyter_text>Helper Functions<jupyter_code>def read_and_process_data(dataset):
    dataset_dir = '../data/ctu/'
    if os.path.exists(dataset_dir + 'no_background_' + dataset + '.pkl'):
        data = pd.read_pickle(
            dataset_dir + 'no_background_' + dataset + '.pkl')  # if the data without the background are there, load them

    else:
        # read the data in chunks due to their large size - uncomment the following lines if you want to read them again
        # and store them in a pickle
        preprocess_bidirectional_data(dataset_dir + dataset)
        dateparse = lambda x: pd.datetime.strptime(x, '%Y/%m/%d %H:%M:%S.%f')
        data = pd.concat(
            remove_background(chunk) for chunk in pd.read_csv(dataset_dir + dataset + '_v2',
                                                              chunksize=100000, delimiter=',',
                                                              parse_dates=['date'], date_parser=dateparse))
        data.to_pickle(dataset_dir + 'no_background_' + dataset + '.pkl')

        # ## Initial preprocessing of the data
        # resetting indices for data
    data = data.reset_index(drop=True)
    dates = data['date'].tolist()
    time_diff = [int(abs(date_diff(s, t))) for s, t in zip(dates[1:], dates)]
    time_diff = [np.mean(time_diff)] + time_diff
    # parse packets and bytes as integers instead of strings
    data['time_diff'] = time_diff
    # parse packets and bytes as integers instead of strings
    data['packets'] = data['packets'].astype(int)
    data['duration'] = data['duration'].astype(float)
    data['src_bytes'] = data['src_bytes'].astype(int)
    data['dst_bytes'] = data['dst_bytes'].astype(int)
    data['total_bytes'] = data['total_bytes'].astype(int)
    data['time_diff'] =  data['time_diff'].astype(int)

    # add the numerical representation of the categorical data
    data['protocol_num'] = pd.Categorical(data['protocol'], categories=data['protocol'].unique()).codes
    # data['state_num'] = pd.Categorical(data['state'], categories=data['state'].unique()).codes
    data['direction_num'] = pd.Categorical(data['direction'], categories=data['direction'].unique()).codes

    return data
def make_bar_graphs(x, y, feature):
    """
    Function for visualising the difference between numerical features (mainly packtes and bytes) for infected and
    normal hosts
    :param x: the type of the hosts
    :param y: the numerical values
    :param feature: the type of the feature
    :return: creates the wanted plot
    """
    plt.figure()
    y_pos = np.arange(len(x))
    plt.bar(y_pos, y, align='center', alpha=0.5, color=[mcolors.TABLEAU_COLORS['tab:blue'],
                                                        mcolors.TABLEAU_COLORS['tab:red']])
    plt.xticks(y_pos, x)
    plt.xlabel('Type of host')
    plt.ylabel(feature)
    plt.title('Average number of %s' % feature)
#     plt.show()
    plt.savefig('plots/barplot_%s.png' % feature)
def make_barplot(data, feature):
    """
    Function for visualising the difference between categorical features for infected and normal hosts
    :param data: the dataframe containing the data
    :return:creates the wanted plot creates the wanted plot
    """
    plt.figure()
    feature_counts = (data.groupby(['is_infected'])[feature].value_counts(normalize=True).rename('percentage').mul(100)
                         .reset_index().sort_values(feature))
    ax = sns.barplot(x=feature, y='percentage', data=feature_counts, hue='is_infected',
                     palette={0: mcolors.TABLEAU_COLORS['tab:blue'], 1: mcolors.TABLEAU_COLORS['tab:red']})
    handles, _ = ax.get_legend_handles_labels()
    ax.legend(handles, ['Normal Hosts', 'Infected host'], loc=0)
    plt.xlabel("%s type" % feature)
    plt.ylabel("Percentage of occurrences")
    plt.grid()
#     plt.show()
    plt.savefig('plots/barplot_%s.png' % feature)
continuous_features = ['direction_num', 'protocol_num', 'duration', 'packets', 
                       'src_bytes', 'dst_bytes', 'total_bytes', 'time_diff', 'stos', 'dtos']
scenarios = range(42,55)
count_dict = defaultdict(lambda: defaultdict(int))
categorical_count_dict = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))


for s in scenarios:
    infected_ips = ip_dict['scenario_{}'.format(s)]['infected_ips']
    normal_ips = ip_dict['scenario_{}'.format(s)]['normal_ips']
    dataset_name = dataset = 'CTU-Malware-Capture-Botnet-{}'.format(s)
    data = read_and_process_data(dataset)
    
    infected = data[data['src_ip'].isin(infected_ips)] 
    infected = infected.reset_index()

    normal= data[data['src_ip'].isin(normal_ips)]
    normal = normal.reset_index()
    
    plot_data = pd.concat([infected, normal], ignore_index=True)
    for index, ip in enumerate(plot_data['src_ip']):
        val = 1 if ip in infected_ips else 0
        plot_data.at[index, 'is_infected'] = val
   
    
    
    for feature in continuous_features:
        count_dict[feature]['normal'] += normal[feature].mean()
        count_dict[feature]['infected'] += infected[feature].mean()
        
for feature in continuous_features:
    for t in ['normal', 'infected']:
        count_dict[feature][t] /= 13
        
# for feature in ['protocol', 'direction']:
#     for t in ['normal', 'infected']:
#         for k in categorical_count_dict[feature][t]:
            
continuous_features = ['direction_num', 'protocol_num', 'duration', 'packets', 
                       'src_bytes', 'dst_bytes', 'total_bytes', 'time_diff', 'stos', 'dtos']
scenarios = range(42,55)
categorical_count_dict = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))

all_plot_data = pd.DataFrame()
for s in scenarios:
    infected_ips = ip_dict['scenario_{}'.format(s)]['infected_ips']
    normal_ips = ip_dict['scenario_{}'.format(s)]['normal_ips']
    dataset_name = dataset = 'CTU-Malware-Capture-Botnet-{}'.format(s)
    data = read_and_process_data(dataset)
    
    infected = data[data['src_ip'].isin(infected_ips)] 
#     infected = data[data['src_ip'] == infected_ips[0]] 
    infected = infected.reset_index()

    normal= data[data['src_ip'].isin(normal_ips)]
#     normal = data[data['src_ip'] == normal_ips[0]] 
    normal = normal.reset_index()
    
    plot_data = pd.concat([infected, normal], ignore_index=True)
    for index, ip in enumerate(plot_data['src_ip']):
        val = 1 if ip in infected_ips else 0
        plot_data.at[index, 'is_infected'] = val
    all_plot_data = pd.concat([all_plot_data, plot_data], ignore_index=True)
   
    
infected = all_plot_data[all_plot_data['is_infected']==1]
infected = infected.reset_index()

normal = all_plot_data[all_plot_data['is_infected']==0]
normal = normal.reset_index()


with pd.option_context('display.max_rows', None, 'display.max_columns', None):
    print('---------------- Stats for infected host ----------------')
    display(HTML(infected[continuous_features].describe().to_html()))
    print('---------------- Stats for normal hosts ----------------')
    display(HTML(normal[continuous_features].describe().to_html()))<jupyter_output>---------------- Stats for infected host ----------------<jupyter_text>Check statistics of the data for the infected and the normal hosts to decide on the features for discretization<jupyter_code># preprocess the infected and normal data and plot possible feature candidates
# plot_data = pd.concat([infected, normal], ignore_index=True)
# for index, ip in enumerate(plot_data['src_ip']):
#     val = 1 if ip == infected_ip else 0
#     plot_data.at[index, 'is_infected'] = val

make_barplot(all_plot_data, 'protocol')
make_barplot(all_plot_data, 'direction')
for feature in continuous_features:
    make_bar_graphs(['Normal Hosts', 'Infected Host'], [count_dict[feature]['normal'], count_dict[feature]['infected']], feature)

print('plots created')<jupyter_output>plots created
{'fraction_non_alphanumeric': 0.14698795180722893, 'fraction_numerical': 0.004565630944831959, 'mean_word_length': 3.532183908045977, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '441843', 'license': 'ECL-2.0', 'n_tokens_mistral': 2496, 'n_tokens_neox': 2379, 'n_words': 545, 'path': 'data_exploration/feature_statistics_all.ipynb', 'repo_name': 'RafailSkoulos17/netflow_classification', 'size': '7885'}