<jupyter_start><jupyter_text>Should this be done?for host in onlineEM.hosts:    tm = onlineEM.hosts[host]['transition_matrix']    for l in  tm:        l[l==0] = 0.00001Or this?for host in onlineEM.hosts:    tm = onlineEM.hosts[host]['transition_matrix']    for i, point in enumerate(onlineEM.hosts[host]['points_per_cluster']):        if point == 0:            tm[i] = np.zeros(onlineEM.m)<jupyter_code># Percentage of times each states leads to the same state
# Approximately

total = 0
same = 0

total_except_zero = 0
same_except_zero = 0
for host in onlineEM.hosts:
    tm = onlineEM.hosts[host]['transition_matrix']
    points_per_cluster = onlineEM.hosts[host]['points_per_cluster']
    total += sum(points_per_cluster)
    total_except_zero += sum(points_per_cluster[1:])
    i = 0
    for p, tvector in zip(points_per_cluster, tm):
        same += tm[i][i] * p
        if i != 0:
            same_except_zero += tm[i][i] * p
        i += 1

    
print('Percentage of times the next state is the same of the previous' , same / total * 100, '%')
print('Percentage of times the next state is the same of the previous not regarding zero traffic', 
      same_except_zero / total_except_zero * 100, '%')<jupyter_output>Percentage of times the next state is the same of the previous 70.4634250851 %
Percentage of times the next state is the same of the previous not regarding zero traffic 29.4609448236 %<jupyter_text>Initialize hmm<jupyter_code>import numpy as np

m = onlineEM.m

initialize = np.eye(m)

for i in range(m):
    for j in range(m):
        if i != j:
            initialize[i][j] = 0.5
            
    initialize[i] /= (1 + 0.5 * (m - 1))
for host in onlineEM.hosts:
    tm = onlineEM.hosts[host]['transition_matrix']
    for i, points in enumerate(onlineEM.hosts[host]['points_per_cluster']):
        tm[i] = tm[i] * (points / (points + 1)) + initialize[i] / (points + 1)<jupyter_output><empty_output><jupyter_text>Running the kmeans algorithm<jupyter_code>from sklearn.externals import joblib

kmeans = joblib.load('pkl/kmeans_20.pkl')
import matplotlib.pyplot as plt 
from pylab import rcParams

rcParams['font.size'] = 16<jupyter_output><empty_output><jupyter_text>Categorize hosts<jupyter_code>from klkmeans import hmm_distance

distances = []
for host in onlineEM.hosts:
    distances.append([host, hmm_distance(onlineEM.hosts[host]['transition_matrix'], kmeans.centers[kmeans.assignments[host]])])
from plots import plot_points_host<jupyter_output><empty_output><jupyter_text>Here are the mst anomalous hosts<jupyter_code># hosts with the higher distance respective to their kmeans center
sorted(distances, key=lambda x: -x[1])[:10]
#find best parameters!!!

def find_percentage(sorted_distances, every=range(10,100,10)):
    total = len(sorted_distances)
    distances = {}
    for k in every:
        distances[k] = sorted_distances[1 - int(total*k/100)][1]
    return distances
from pylab import rcParams
from matplotlib import pyplot as plt

colors = ['blue', 'red', 'green', 'yellow', 'black']
rcParams['font.size'] = 16
rcParams['figure.figsize'] = 16, 9

sorted_distances = sorted(distances, key=lambda x: -x[1])

ranges = np.arange(1,100.1,0.1)
show_ranges = range(5,101,5)

percentages = list(find_percentage(sorted_distances, every=ranges).values())
  
plt.plot(list(ranges), percentages)
plt.xlabel('Percentage of hosts')
plt.ylabel('Distance of host\'s transition matrix to its closest center')
plt.xticks(show_ranges)
plt.grid() 
plt.show()
# Find the average distances of hosts from their center in each cluster
import numpy as np
from klkmeans import hmm_distance


average_distances = {}
for k,v in kmeans.assignments.items():
    # manually inserted anomalous host
    if k.startswith("A"):
        continue
    if v in average_distances:
        average_distances[v].append(hmm_distance(onlineEM.hosts[k]['transition_matrix'], kmeans.centers[v]))
    else:
        average_distances[v] = []
        average_distances[v].append(hmm_distance(onlineEM.hosts[k]['transition_matrix'], kmeans.centers[v]))
        
for k,v in sorted(average_distances.items()):
    print('Average distances from enter  {:2d} for the hosts with this as their closest center is {:.3f}'.format(k, 
                                                                                                                 sum(v)/len(v)))
from pylab import rcParams
from matplotlib import pyplot as plt
import numpy as np
from collections import Counter
import matplotlib.patches as mpatches

import seaborn as sns

colors = ['blue', 'red', 'green', 'yellow']
styles = ['-','--',':','-.']
rcParams['font.size'] = 16

def plot_points_host(data, hostData=None, em=None):
    rcParams['figure.figsize'] = 16, 9
    data_hashable = [tuple(x) for x in data]
    total_points = len(data_hashable) / 10000

    values = np.vstack([list(x) for x in list(Counter(data_hashable).keys())])
    counts = np.array(list(Counter(data_hashable).values()))

    for i in range(len(values)):
        plt.scatter(values[i][0], values[i][1], s=counts[i]/total_points, color='blue')
        
    patches=[mpatches.Patch(color='blue', label='Data points')]
    
    if hostData is not None:    
        data_hashable = [tuple(x) for x in hostData]
        total_points = len(data_hashable) / 100

        values = np.vstack([list(x) for x in list(Counter(data_hashable).keys())])
        counts = np.array(list(Counter(data_hashable).values()))
        for i in range(len(values)):
            plt.scatter(values[i][0], values[i][1], s=counts[i]*100/total_points, color='orange')
            
        patches.append(mpatches.Patch(color='orange', label='Host data'))
        
    if em is not None:
        for i, lambda_i in enumerate(em.lambdas):
            plt.scatter(lambda_i[0], lambda_i[1], s=em.gammas[i]*2500, linewidth=4, color='red', marker='x')

        patches.append(mpatches.Patch(color='red', label='Distribution centers'))
    
    plt.legend(handles=patches, fontsize=18)
    plt.ylabel('average number of bytes')
    plt.xlabel('number of flows')
    plt.show()
host = sorted_distances[0][0]
print('Host', host, 'has the most anomalous behavior')
plot_points_host(groupped_data.values[:, :2], hostData=groupped_data[groupped_data['source computer'] == host].values[:, :2],
                em=onlineEM)
from pylab import rcParams
from matplotlib import pyplot as plt
import numpy as np
from collections import Counter
import matplotlib.patches as mpatches

import seaborn as sns


colors = ['blue', 'red', 'green', 'yellow']
styles = ['-','--',':','-.']
rcParams['font.size'] = 16
# HOST and clusters
%matplotlib inline

def plot_transition_matrix(em, kmeans, host):
    rcParams['figure.figsize'] = 16, 9

    plt.subplot(1,2,1)
    ax = sns.heatmap(em.hosts[host]['transition_matrix'])
    plt.title('Host transition matrix')
    
    plt.subplot(1,2,2)
    ax = sns.heatmap(kmeans.centers[kmeans.assignments[host]])
    plt.title('Kmeans center transition matrix for center ' + str(kmeans.assignments[host]))
    plt.show()
onlineEM.lambdas
onlineEM.hosts[host]['points_per_cluster']
plot_transition_matrix(onlineEM, kmeans, host)
import collections
    
centers = {}
for host in onlineEM.hosts:
    if kmeans.assignments[host] in centers:
        centers[kmeans.assignments[host]] += 1 
    else:
        centers[kmeans.assignments[host]] = 1


participations = collections.OrderedDict(sorted(centers.items()))
print(participations)
rcParams['figure.figsize'] = 16, 35
    

for i in range(len(kmeans.centers)):
    plt.subplot(len(kmeans.centers) / 2,2,i+1)
    ax = sns.heatmap(kmeans.centers[i])
    if i in participations:
        num = str(participations[i])
    else:
        num = str(0)
    plt.title('Kmeans center ' + str(i) + ' with a total of ' + num + ' hosts')

plt.show()
#calculate_likelihood_em(onlineEM, groupped_data1.values[:,[0,1,3]], kmeans, take_mean=False, weight=0.5, verbose=1)
#calculate_likelihood_em(onlineEM, groupped_data1.values[:,[0,1,3]], kmeans, take_mean=True, weight=0.2, verbose=1)<jupyter_output><empty_output><jupyter_text>Create random traffic for a new host<jupyter_code>def percentage_anomalous(sorted_distances, value):
    total = len(sorted_distances)
    values_smaller = len(np.where(np.vstack(np.array(sorted_distances, dtype=object))[:, 1] < value)[0])
    return values_smaller / total
import random as ran

def random_vector(size=7):
    vec = [ran.random() for i in range(size)]
    return np.array(vec) / sum(vec)

def random_transition_matrix(size=7):
    return np.vstack([random_vector(size=size) for i in range(size)])
t = []
size = 50000

test_df = groupped_data.head(size)
epochs = 1500


for i in range(30):
    anomalous_host = 'ΒΒΒBBBΒ' + str(i)
    
    data = []
    for i in range(epochs):
        #flows = ran.randint(1,50)
        #average_bytes = ran.randint(1,50)
        values = ran.choice(test_df.values[:, :2])
        data.append([values[0], values[1], anomalous_host])

    for data_point in data:
        onlineEM.update_host(data_point)
        
    tm = onlineEM.hosts[anomalous_host]['transition_matrix']
    for i, points in enumerate(onlineEM.hosts[anomalous_host]['points_per_cluster']):
        tm[i] = tm[i] * (points / (points + 1)) + initialize[i] / (points + 1)
        
    distances_for_anomalous_host = []
    for i in range(len(kmeans.centers)):
        distances_for_anomalous_host.append([i, hmm_distance(onlineEM.hosts[anomalous_host]['transition_matrix'], 
                                             kmeans.centers[i]) ])
    distances_for_anomalous_host.sort(key=lambda x: x[1])
    t.append(distances_for_anomalous_host[0][1])
    v = average_distances[distances_for_anomalous_host[0][0]]
    print(distances_for_anomalous_host[0][1], sum(v)/len(v))
    
value = sum(t)/len(t)
print('FINAL')
print(value)
print(percentage_anomalous(sorted_distances, value))
# RANDOM COMBINATION OF FLOWS AND NUMBER OF BYTES
import random as ran
import random

random_flows = list(set(groupped_data['number of flows'].values))
random_average_bytes = list(set(groupped_data['mean(byte count)'].values))

anomalous_host = 'A70'
epochs = 1500

data = []
for i in range(epochs):
    #flows = ran.randint(1,50)
    #average_bytes = ran.randint(1,50)
    flows = random.choice(random_flows)
    average_bytes = random.choice(random_average_bytes)
    data.append([flows, average_bytes, anomalous_host])

    
for data_point in data:
    onlineEM.update_host(data_point)
#RANDOM VALUE ENCOUNTERED IN THE SET OF GROUPED_DATA
import random as ran
import random

size = 10000

test_df = groupped_data.head(size)
unique_values = np.vstack({tuple(row) for row in test_df.values[:,:2]})

anomalous_host = 'A71'
epochs = 1500

data = []
for i in range(epochs):
    values = random.choice(unique_values)
    data.append([values[0], values[1], anomalous_host])

    
for data_point in data:
    onlineEM.update_host(data_point)
#RANDOM VALUE ENCOUNTERED IN GROUPED_DATA
import random as ran
import random

size = 10000

test_df = groupped_data.head(size)

anomalous_host = 'A72'
epochs = 1500

data = []
for i in range(epochs):
    values = random.choice(test_df.values[:,:2])
    data.append([values[0], values[1], anomalous_host])

    
for data_point in data:
    onlineEM.update_host(data_point)
tm = onlineEM.hosts[anomalous_host]['transition_matrix']
for i, points in enumerate(onlineEM.hosts[anomalous_host]['points_per_cluster']):
    tm[i] = tm[i] * (points / (points + 1)) + initialize[i] / (points + 1)
distances_for_anomalous_host = []
for i in range(len(kmeans.centers)):
    distances_for_anomalous_host.append([i, hmm_distance(onlineEM.hosts[anomalous_host]['transition_matrix'], 
                                         kmeans.centers[i]) ])
distances_for_anomalous_host.sort(key=lambda x: x[1])
distances_for_anomalous_host
onlineEM.hosts[anomalous_host]['points_per_cluster']
kmeans.assignments[anomalous_host] = distances_for_anomalous_host[0][1]
plot_transition_matrix(onlineEM, kmeans, anomalous_host)<jupyter_output><empty_output><jupyter_text>Number of clusters to choose<jupyter_code>from klkmeans import hmm_distance
import copy

results = {}
kmeans_all = {}
clusters = list(range(31,41))
for n_clusters in clusters:
    print('Now running fot number of clusters', n_clusters)
    kmeans = kl_kMeans(onlineEM, n_clusters=n_clusters, n_iters=15, n_runs=10)
    C, assign = kmeans.run(verbose=0, initialization='k++')
     
    distances = []
    for host in onlineEM.hosts:
        distances.append([host, hmm_distance(onlineEM.hosts[host]['transition_matrix'], kmeans.centers[assign[host]])])
        
    sorted_distances = sorted(distances, key=lambda x: -x[1])
    results[n_clusters] = sorted_distances.copy()
    kmeans_all[n_clusters] = copy.deepcopy(kmeans)
ranges = range(10,100,5)

percentages = []
for n_clusters in clusters:
    percentages.append(list(find_percentage(results[n_clusters], every=ranges).values()))
from pylab import rcParams
from matplotlib import pyplot as plt

colors = ['blue', 'red', 'green', 'yellow', 'black']
rcParams['font.size'] = 16
rcParams['figure.figsize'] = 16, 9

show_clusters = [10, 15, 20, 25, 30]
for percentage, color, n_clusters in zip(percentages, colors, show_clusters):
    plt.plot(ranges, percentage, label='Number of cluster: ' + str(n_clusters))

plt.xlabel('Percentage of hosts')
plt.ylabel('Distance of host\'s transition matric to its closest center smaller than')
plt.xticks(ranges)
plt.grid()
plt.legend()    
plt.show()<jupyter_output><empty_output><jupyter_text>Inertia plot<jupyter_code>from sklearn.externals import joblib

joblib.dump(kmeans_all, 'kmeans_all1.pkl')
#kmeans_all = joblib.load('kmeans_all.pkl')
from klkmeans import hmm_distance

results = {}
for n_clusters, kmeans in kmeans_all.items():
    distances = []
    for host in onlineEM.hosts:
        distances.append(hmm_distance(onlineEM.hosts[host]['transition_matrix'], kmeans.centers[kmeans.assignments[host]]))
        
    sorted_distances = sorted(distances)
    results[n_clusters] = sorted_distances.copy()
    
    #print(kmeans._cost(kmeans.centers, kmeans.assignments))
# OLD SILHOUETTE !!!!!!!!!!!!!!!!!!!!

from sklearn.metrics import silhouette_score
from klkmeans import hmm_distance

sil = []
for n_clusters, kmeans in kmeans_all.items():
    means_intra_distances = {}
    for i in range(len(kmeans.centers)):
        matrices = kmeans._transition_matrices_for_cluster(i, kmeans.assignments)
        
        cost = 0
        for transition_matrix in matrices:
            cost += hmm_distance(transition_matrix, kmeans.centers[i])
        
        means_intra_distances[i] = cost / len(matrices)
    
    total_silhouette_score = 0
    for host in onlineEM.hosts:
        distances_for_host = []
        for i in range(len(kmeans.centers)):
            distances_for_host.append([i, hmm_distance(onlineEM.hosts[host]['transition_matrix'], 
                                                 kmeans.centers[i]) ])
        distances_for_host.sort(key=lambda x: x[1])
        
        b = distances_for_host[1][1]
        a = means_intra_distances[kmeans.assignments[host]]
        
        total_silhouette_score += (b - a) / max(a, b)
        
    
    sil.append(total_silhouette_score)
    print('kmeans', len(kmeans.centers), 'silhouette', total_silhouette_score)<jupyter_output><empty_output><jupyter_text>"""kmeans 10 silhouette 1208.11181954kmeans 11 silhouette 1059.74116867kmeans 12 silhouette 1081.55146209kmeans 13 silhouette 1131.6618338kmeans 14 silhouette 1215.26372084kmeans 15 silhouette 1201.73655985kmeans 16 silhouette 1200.93941438kmeans 17 silhouette 1220.80162766kmeans 18 silhouette 1262.53047367kmeans 19 silhouette 1253.53400928kmeans 20 silhouette 1293.26855628kmeans 21 silhouette 1237.35126978kmeans 22 silhouette 1263.83071776kmeans 23 silhouette 1332.39862265kmeans 24 silhouette 1261.26918699kmeans 25 silhouette 1109.14159873kmeans 26 silhouette 1326.28038329kmeans 27 silhouette 1162.17096983kmeans 28 silhouette 1134.1026461kmeans 29 silhouette 1347.94228013kmeans 30 silhouette 1350.14320671"""<jupyter_code>sil = np.array(sil) / len(onlineEM.hosts)
for k,v in kmeans_all.items():
    print(k, v._cost(v.centers, c.assignments))
from sklearn.metrics import silhouette_score
from klkmeans import hmm_distance

sil2 = []
for n_clusters, kmeans in kmeans_all.items(): 
    print('Running for n_clusters', n_clusters)
    total_silhouette_score = 0
    j = 0
    for host in onlineEM.hosts:
        j += 1
        if j % 100 == 0:
            print('Done with', j)
        
        distances = []
        for host1 in onlineEM.hosts:
            if host1 == host:
                continue
            
            distances.append([kmeans.assignments[host1], host1, hmm_distance(onlineEM.hosts[host]['transition_matrix'], 
                                                                             onlineEM.hosts[host1]['transition_matrix'])])
        
        mean_distances = np.zeros(len(kmeans.centers))
        for i in range(len(kmeans.centers)):
            total = 0
            for dis in distances:
                if dis[0] == i:
                    mean_distances[i] += dis[2]
                    total += 1
            mean_distances[i] /= total
            
        a = mean_distances[kmeans.assignments[host]]
        
        mean_distances[kmeans.assignments[host]] = 1000000
        b = np.min(mean_distances)
        
        total_silhouette_score += (b - a) / max(a, b)
            
    sil2.append(total_silhouette_score)
    print('kmeans', len(kmeans.centers), 'silhouette', total_silhouette_score)
    
sil2 = np.array(sil2)
sil2 /= len(onlineEM.hosts)
sil2
import matplotlib.pyplot as plt
%matplotlib inline
from pylab import rcParams

rcParams['figure.figsize'] = 16, 9
rcParams['font.size'] = 15


inertia = []
show_clusters = list(range(31, 36))

for n_clusters in show_clusters:
    distances = results[n_clusters]
    inertia.append(sum(distances))

plt.subplot(1,2,1)

plt.xlabel('Number of clusters')
plt.ylabel('inertia')
plt.xticks(list(range(10, 31, 2)))
plt.grid()
plt.plot(show_clusters, inertia)


plt.subplot(1,2,2)
plt.xlabel('Number of clusters')
plt.ylabel('silhouette coefficient')
plt.xticks(list(range(10, 31, 2)))
plt.grid()
plt.plot(show_clusters, sil2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1236139181733763, 'fraction_numerical': 0.034030698639864534, 'mean_word_length': 3.863974495217853, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '603576', 'license': 'MIT', 'n_tokens_mistral': 6793, 'n_tokens_neox': 6171, 'n_words': 1346, 'path': 'diploma/KL-Kmeans.ipynb', 'repo_name': 'SotirisAnagnostidis/onlineAnomalyDetection', 'size': '18307'}