<jupyter_start><jupyter_text>Self Organizing Systems - Metro Maps Vizualisation Stella Mahler: 11936014 <NAME>: 51829460  Provided Code  SOMToolBox_Parse class<jupyter_code>import pandas as pd
import numpy as np
import gzip


class SOMToolBox_Parse:
    
    def __init__(self, filename):
        self.filename = filename
    
    def read_weight_file(self,):
        df = pd.DataFrame()
        if self.filename[-3:len(self.filename)] == '.gz':
            with gzip.open(self.filename, 'rb') as file:
                df, vec_dim, xdim, ydim = self._read_vector_file_to_df(df, file)
        else:
            with open(self.filename, 'rb') as file:
                df, vec_dim, xdim, ydim = self._read_vector_file_to_df(df, file)

        file.close()            
        return df.astype('float64'), vec_dim, xdim, ydim


    def _read_vector_file_to_df(self, df, file):
        xdim, ydim, vec_dim, position = 0, 0, 0, 0
        for byte in file:
            line = byte.decode('UTF-8')
            if line.startswith('$'):
                xdim, ydim, vec_dim = self._parse_vector_file_metadata(line, xdim, ydim, vec_dim)
                if xdim > 0 and ydim > 0 and len(df.columns) == 0:
                    df = pd.DataFrame(index=range(0, ydim * xdim), columns=range(0, vec_dim))
            else:
                if len(df.columns) == 0 or vec_dim == 0:
                    raise ValueError('Weight file has no correct Dimensional information.')
                position = self._parse_weight_file_data(line, position, vec_dim, df)
        return df, vec_dim, xdim, ydim


    def _parse_weight_file_data(self, line, position, vec_dim, df):
        splitted=line.split(' ')
        try:
            df.values[position] = list(np.array(splitted[0:vec_dim]).astype(float))
            position += 1
        except: raise ValueError('The input-vector file does not match its unit-dimension.') 
        return  position


    def _parse_vector_file_metadata(self, line, xdim, ydim, vec_dim):
        splitted = line.split(' ')
        if splitted[0] == '$XDIM':      xdim = int(splitted[1])
        elif splitted[0] == '$YDIM':    ydim = int(splitted[1])
        elif splitted[0] == '$VEC_DIM': vec_dim = int(splitted[1])
        return xdim, ydim, vec_dim<jupyter_output><empty_output><jupyter_text>SomViz class<jupyter_code>import numpy as np
from scipy.spatial import distance_matrix, distance
from ipywidgets import Layout, HBox, Box, widgets, interact
import plotly.graph_objects as go


from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>")) 

class SomViz:
    
    def __init__(self, weights=[], m=None, n=None):
        self.weights = weights
        self.m = m
        self.n = n



    def component_plane(self, som_map=None, component=0, color="Viridis", interp = "best", title=""):
        if som_map==None: return self.plot(self.weights[:,component].reshape(-1,self.n), color=color, interp=interp, title=title)   
        else:  som_map.data[0].z = self.weights[:,component].reshape(-1,n)

   
    def project_data(self,som_m=None, idata=[], title=""):

        data_y = []
        data_x = []
        for v in idata:
            position =np.argmin(np.sqrt(np.sum(np.power(self.weights - v, 2), axis=1)))
            x,y = position % self.n, position // self.n
            data_x.extend([x])
            data_y.extend([y])
            
        if som_m!=None: som_m.add_trace(go.Scatter(x=data_x, y=data_y, mode = "markers", marker_color='rgba(255, 255, 255, 0.8)',))
    
    def time_series(self, som_m=None, idata=[], wsize=50, title=""): #not tested
             
        data_y = []
        data_x = [i for i in range(0,len(idata))]
        
        data_x2 = []
        data_y2 = []
        
        qmin = np.Inf
        qmax = 0
        
        step=1
        
        ps = []
        for v in idata:
            matrix = np.sqrt(np.sum(np.power(self.weights - v, 2), axis=1))
            position = np.argmin(matrix)
            qerror = matrix[position]
            if qmin>qerror: qmin = qerror
            if qmax<qerror: qmax = qerror
            ps.append((position, qerror))
       
        markerc=[]    
        for v in ps:
            data_y.extend([v[0]])
            rez = v[1]/qmax
 
            markerc.append('rgba(0, 0, 0, '+str(rez)+')') 
            
            x,y = v[0] % self.n, v[0] // self.n 
            if    x==0: y = np.random.uniform(low=y, high=y+.1)
            elif  x==self.m-1: y = np.random.uniform(low=y-.1, high=y)
            elif  y==0: x = np.random.uniform(low=x, high=x+.1)
            elif  y==self.n-1: x = np.random.uniform(low=x-.1, high=x)
            else: x,y = np.random.uniform(low=x-.1, high=x+.1), np.random.uniform(low=y-.1, high=y+.1)                           
            
            data_x2.extend([x])
            data_y2.extend([y]) 
    
        ts_plot = go.FigureWidget(go.Scatter(x=[], y=[], mode = "markers", marker_color=markerc, marker=dict(colorscale='Viridis', showscale=True, color=np.random.randn(500))))
        ts_plot.update_xaxes(range=[0, wsize])       

        
        ts_plot.data[0].x, ts_plot.data[0].y = data_x, data_y
        som_m.add_trace(go.Scatter(x=data_x2, y=data_y2, mode = "markers",))
  
        som_m.layout.height = 500
        ts_plot.layout.height = 500
        som_m.layout.width = 500
        ts_plot.layout.width = 1300
        
        return HBox([go.FigureWidget(som_m), go.FigureWidget(ts_plot)])
    

    
    def plot(self, matrix, color="Viridis", interp = "best", title=""):
        return go.FigureWidget(go.Heatmap(z=matrix, zsmooth=interp, showscale=False, colorscale=color), layout=go.Layout(width=700, height=700,title=title, title_x=0.5,))<jupyter_output><empty_output><jupyter_text>SomToolBox Vizualisations  Left: SomToolBox Right: Python<jupyter_code>import matplotlib.pyplot as plt
#10clusters40x20
img1 = plt.imread('viz/10clusters/MetroMap40x20.png')
img2 = plt.imread('viz/10clusters/MetroMap40x20.png')
#10clusters40x20_comp1
img3 = plt.imread('viz/10clusters/MetroMap40x20_Comp1.png')
img4 = plt.imread('viz/10clusters/MetroMap40x20_Comp1.png')


#10clusters100x60
img5 = plt.imread('viz/10clusters/MetroMap100x60_6bins.png')
img6 = plt.imread('viz/10clusters/MetroMap100x60_6bins.png')
#10clusters100x60_comp1
img7 = plt.imread('viz/10clusters/MetroMap100x60_6bins_Comp1.png')
img8 = plt.imread('viz/10clusters/MetroMap100x60_6bins_Comp1.png')

#chainlink40x20
img9 = plt.imread('viz/chainlink/MetroMap40x20_6bins.png')
img10 = plt.imread('viz/chainlink/MetroMap40x20_6bins.png')
#chainlink40x20_comp1a
img11 = plt.imread('viz/chainlink/MetroMap40x20_6bins_comp1.png')
img12 = plt.imread('viz/chainlink/MetroMap40x20_6bins_comp1.png')


#chainlink100x60
img13 = plt.imread('viz/chainlink/MetroMap100x60_6bins.png')
img14 = plt.imread('viz/chainlink/MetroMap100x60_6bins.png')
#chainlink100x60_comp1
img15 = plt.imread('viz/chainlink/MetroMap100x60_6bins_comp1.png')
img16 = plt.imread('viz/chainlink/MetroMap100x60_6bins_comp1.png')

#iris
img17 = plt.imread('viz/iris/MetroMap_6bins.png')
img18 = plt.imread('viz/iris/MetroMap_6bins.png')
#iris_comp1
img19 = plt.imread('viz/iris/MetroMap_6bins_comp1.png')
img20 = plt.imread('viz/iris/MetroMap_6bins_comp1.png')




#MetroMap40x20_Comp1.png
images = [img1, img2,img3, img4,img5, img6,img7, img8, img9, img10 ,img11, img12,img13, img14,img15, img16,img17, img18,img19, img20]

NUM_ROWS = 10
IMGs_IN_ROW = 2

f, ax_arr = plt.subplots(NUM_ROWS, IMGs_IN_ROW, figsize=(18,100))

for j, row in enumerate(ax_arr):
    for i, ax in enumerate(row):
        ax.imshow(images[j*IMGs_IN_ROW+i])
        ax.set_title(f'image {j*IMGs_IN_ROW+i+1}')

title = 'displaying images matrix'
f.suptitle(title, fontsize=16)
plt.show() 
##########################################
######## read from SOMToolBox ############
##########################################
trainedmap = SOMToolBox_Parse('iris.vec')
idata, idim, idata_x, idata_y = trainedmap.read_weight_file()

smap = SOMToolBox_Parse('iris.wgt.gz')
smap, sdim, smap_x, smap_y = smap.read_weight_file()

# Visualizaton
viz_SOMToolBox = SomViz(smap.values.reshape(-1,sdim), smap_y, smap_x)
#um2 = viz_SOMToolBox.umatrix(color='viridis', interp=False, title='U-matrix SOMToolBox')
um3 = viz_SOMToolBox.component_plane(component=0, color='viridis', interp=False, title='component_plane0 SOMToolBox')
um4 = viz_SOMToolBox.component_plane(component=1, color='viridis', interp=False, title='component_plane1 SOMToolBox')

um5 = viz_SOMToolBox.component_plane(component=2, color='viridis', interp=False, title='component_plane2 SOMToolBox')
um6 = viz_SOMToolBox.component_plane(component=3, color='viridis', interp=False, title='component_plane3 SOMToolBox')



#def umatrix(self, som_map=None, color="Viridis", interp = "best", title=""):
#def component_plane(self, som_map=None, component=0, color="Viridis", interp = "best", title=""):

display(HBox([um3, um4]))
display(HBox([um5, um6]))


trainedmap = SOMToolBox_Parse('iris.vec')
idata, idim, idata_x, idata_y = trainedmap.read_weight_file()

smap = SOMToolBox_Parse('iris.wgt.gz')
smap, sdim, smap_x, smap_y = smap.read_weight_file()

# Visualizaton
viz_SOMToolBox = SomViz(smap.values.reshape(-1,sdim), smap_y, smap_x)

um3 = viz_SOMToolBox.component_plane(component=0, color='viridis', interp="best", title='component_plane0 SOMToolBox')
um4 = viz_SOMToolBox.component_plane(component=1, color='viridis', interp="best", title='component_plane1 SOMToolBox')

smap = SOMToolBox_Parse('iris.wgt.gz')
smap, sdim, smap_x, smap_y = smap.read_weight_file()

weights = smap.values.reshape(-1,sdim)
x = smap_x
y = smap_y
m = 10
n = 10


def component_plane_test(weights, som_map=None, component=0, color="Viridis", interp = "best", title=""):
    if som_map==None: 
        return plot(weights[:,component].reshape(-1,n), color=color, interp=interp, title=title)   
    else:  som_map.data[0].z = weights[:,component].reshape(-1,n)
        
    
def plot(matrix, color="Viridis", interp = "best", title=""):
    return go.FigureWidget(go.Heatmap(z=matrix, zsmooth=interp, showscale=False, colorscale=color), layout=go.Layout(width=700, height=700,title=title, title_x=0.5,))


trainedmap = SOMToolBox_Parse('iris.vec')
idata, idim, idata_x, idata_y = trainedmap.read_weight_file()

smap = SOMToolBox_Parse('iris.wgt.gz')
smap, sdim, smap_x, smap_y = smap.read_weight_file()

# Visualizaton
viz_SOMToolBox = SomViz(smap.values.reshape(-1,sdim), smap_y, smap_x)

um3 = component_plane_test(weights, component=0, color='viridis', interp="best", title='component_plane0 SOMToolBox')
um4 = component_plane_test(weights, component=1, color='viridis', interp="best", title='component_plane1 SOMToolBox')

display(HBox([um3, um4]))
comp_0 = weights[:,0].reshape(-1,n)
comp_1 = weights[:,1].reshape(-1,n)
# discretization

# each component is split into n disjoint ranges
# Schwellwert sind gleichweit entfernte Punkte zwischen dem niedrigsten und höchsten Wert einer bestimmten component plane.

# Parameter:
#component = comp_1
#bins = 3

def discretize(component, bins):
    # get min/max value
    min_value = component.min()
    max_value = component.max()

    # get upper threshold for each region
    upper_threshold = []
    for k in range(bins):
        result = (((k+1) * (max_value - min_value)) / bins ) + min_value ## @ JJ check mal, ob das Sinn macht
        upper_threshold.append(result)
    
    #print(upper_threshold)
    
    # discretize values
    discrete_component = np.digitize(component,bins=upper_threshold, right = True)

    #plot(discrete_component, color="viridis", interp=False, title="discrete component")   
    
    return discrete_component



def plot(matrix,color="Viridis", interp = "best", title=""):

    return go.FigureWidget(go.Heatmap(z=matrix, zsmooth=interp, showscale=False, colorscale=color),layout=go.Layout(width=700, height=700,title=title, title_x=0.5,))
def plot_gravity(matrix, component_lines, color="Viridis", interp = "best", title=""):
    
    lines = []
    for line in component_lines:
        # Convert list of points to dictionary
        x_coordinates = []
        y_coordinates = []
        for point in line:
            x_coordinates.append(point[0])
            y_coordinates.append(point[1])
        
        line_dict = {}
        line_dict["x"] = x_coordinates
        line_dict["y"] = y_coordinates
        
  
        # draw line
        lines.append(go.Scatter(line_dict,
                   mode='lines+markers',
                   #line_color='black', 
                    line_width=2.5))
   
    
  
    # draw heatmap
    heatmap = go.Heatmap(z=matrix, zsmooth=interp, showscale=False, colorscale=color)
    
    lines.append(heatmap)
    
    return go.FigureWidget(data= lines, layout=go.Layout(width=700, height=700,title=title, title_x=0.5,))<jupyter_output><empty_output><jupyter_text>Compute centre of gravity:![image.png](attachment:image.png)<jupyter_code># Region centers
# centres of gravity for individual groups of components
from scipy import ndimage

# parameter
#discrete_component = discrete_component
#bins = len(np.unique(discrete_component)) # Alternative: use bin as parameter

def centre_of_gravity(discrete_component):
    bins = len(np.unique(discrete_component))
    
    centres_of_gravity = []
            
    for region in range(bins):
        no_of_units_in_region = np.count_nonzero(discrete_component == region)        
            
        sum_of_units_in_region = (0,0)
        for x in range(len(discrete_component)):
            for y in range(len(discrete_component[0])):
                if (discrete_component[x][y] == region):
                    sum_of_units_in_region = tuple(map(sum, zip(sum_of_units_in_region, (x,y))))
                    #print(x)
                    #print(y)
                    #print(sum_of_units_in_region)
       
        result = [0,0]
        result[1] = (1 / no_of_units_in_region) * sum_of_units_in_region[0] # TODO JJ
        result[0] = (1 / no_of_units_in_region) * sum_of_units_in_region[1] # TODO JJ
        centres_of_gravity.append(result)    
        

#    sum_of_units_in_region_x_value = 1 #TODO JJ
#    sum_of_units_in_region_y_value = 2 # TODO JJ
        
    
    ##################
    ##################
    
    # temp
#    result = ndimage.measurements.center_of_mass(discrete_component, labels = region)
    #print(region)
    #print(result)

#centres_of_gravity = ndimage.measurements.center_of_mass(discrete_component)
  
#x_center, y_center = np.argwhere(discrete_component==3).sum(0)/count
#print(centres_of_gravity)

################
################


# https://stackoverflow.com/a/36957830
#    for i in range(bins):
#        matches = np.transpose((discrete_component == i).nonzero())
#        result = np.mean(matches, axis=0)
        #print ("The center of cluster {} is at {}".format(i, result))
#        centres_of_gravity.append(result)

    return centres_of_gravity
    #plot_gravity(discrete_component,centres_of_gravity, color="viridis", interp=False, title="title")   

# weights, m, n kann später weg, wenn wir es in die Klasse setzen, muss dann durch self.weights etc. ersetzt werden.

def metro_map (weights, m, n, bins, components= None, base= None):
    
    # consider all components for the case there are no components specified
    if components == None:
        components = range(weights.shape[1])
        
    component_lines = []
    for i in components:
        comp = weights[:,i].reshape(-1,n)
        discrete_component = discretize(comp, bins)
        component_lines.append(centre_of_gravity(discrete_component))
        
    print(component_lines)
    print(discrete_component)
    # TODO
    base = discrete_component
    
    return plot_gravity(base, component_lines, interp=False)
    #return plot(discrete_component, interp = False)
    
        

smap = SOMToolBox_Parse('iris.wgt.gz')
smap, sdim, smap_x, smap_y = smap.read_weight_file()

weights = smap.values.reshape(-1,sdim)
n = smap_x
m = smap_y



viz_SOMToolBox = SomViz(smap.values.reshape(-1,sdim), smap_y, smap_x)
#um2 = viz_SOMToolBox.umatrix(color='viridis', interp=False, title='U-matrix SOMToolBox')
#base = viz_SOMToolBox.component_plane(component=0, color='viridis', interp=False, title='component_plane0 SOMToolBox')


um3 = metro_map(weights,m = m, n = n, components = [0], bins =  4, base = "whatever")
um3<jupyter_output>[[[2.2857142857142856, 1.2380952380952381], [3.595238095238095, 4.5476190476190474], [6.419354838709677, 6.0], [8.666666666666666, 7.833333333333333]]]
[[0 0 0 0 0 0 0 1 1 1]
 [0 0 0 0 0 1 0 1 1 1]
 [0 0 0 0 0 1 1 1 2 2]
 [0 0 1 1 1 1 1 2 2 2]
 [0 1 1 1 1 1 2 2 2 2]
 [1 1 1 1 1 1 1 2 2 2]
 [1 1 1 1 2 2 2 2 2 3]
 [1 1 1 1 2 2 2 2 2 3]
 [1 1 1 1 2 2 2 2 3 3]
 [2 1 1 1 2 2 2 2 3 3]]<jupyter_text><jupyter_code>weights.shape[1]
discrete_component
comp_0
import pandas as pd
import minisom as som
from sklearn import datasets, preprocessing
#interp: False, 'best', 'fast', 
#color = 'viridis': https://plotly.com/python/builtin-colorscales/



#############################
######## miniSOM ############1/0
#############################
m=10
n=10

# Pre-processing 
iris = datasets.load_iris().data
min_max_scaler = preprocessing.MinMaxScaler()
iris = min_max_scaler.fit_transform(iris)

# Train
s = som.MiniSom(m, n, iris.shape[1], sigma=0.8, learning_rate=0.7)
s.train_random(iris, 10000, verbose=False)

# Visualizaton
viz_miniSOM = SomViz(s._weights.reshape(-1,4), m, n)
#um1 = viz_miniSOM.umatrix(color='magma', interp='best', title='U-matrix miniSOM')


##########################################
######## read from SOMToolBox ############
##########################################
trainedmap = SOMToolBox_Parse('iris.vec')
idata, idim, idata_x, idata_y = trainedmap.read_weight_file()

smap = SOMToolBox_Parse('iris.wgt.gz')
smap, sdim, smap_x, smap_y = smap.read_weight_file()

# Visualizaton
viz_SOMToolBox = SomViz(smap.values.reshape(-1,sdim), smap_y, smap_x)
um2 = viz_SOMToolBox.umatrix(color='viridis', interp=False, title='U-matrix SOMToolBox') 

display(HBox([um1, um2]))
import plotly.graph_objects as go

# Create Subplots
fig = go.Figure()

# Set axes properties
fig.update_xaxes(range=[0, 20], zeroline=False)
fig.update_yaxes(range=[0, 20])

#fig.add_trace(go.Scatter(x=[20], y=[20]))

# Add shapes
fig.update_layout(
    shapes=[
        dict(type="circle", xref="x", yref="y",
             x0=5, y0=10, x1=10, y1=15, fillcolor="White",)])
fig.show()


import numpy as np
from numpy import pi, sin, cos
import plotly.graph_objects as go

import math

def ellipse(x_center=0, y_center=0, ax1 = [[1, 0],[0,1]], a=1, b =1,  N=100):
    # x_center, y_center the coordinates of ellipse center
    # ax1 two orthonormal vectors representing the ellipse axis directions
    # a, b the ellipse parameters
    
    t = np.linspace(0, 2*pi, N)
    #ellipse parameterization with respect to a system of axes of directions a1, a2
    xs = a * cos(t)
    ys = b * sin(t)
    #rotation matrix
    R = np.array([ax1[0], ax1[1]]).T
    # coordinate of the  ellipse points with respect to the system of axes [1, 0], [0,1] with origin (0,0)
    xp, yp = np.dot(R, [xs, ys])
    x = xp + x_center 
    y = yp + y_center
    return x, y
    
#EXAMPLE
fig = go.Figure()
x_center=10
y_center=10

#a = height
#b = width
#ax1, = angle

upwards = [[0,1],[1,0]]
downwards = [[0,-1],[-1,0]]
right = [[1,0],[0,-1]]
left = [[-1,0],[0,1]]

topright = [[math.sqrt(2)/2, math.sqrt(2)/2],[math.sqrt(2)/2,-math.sqrt(2)/2]]
topleft = [[-math.sqrt(2)/2, math.sqrt(2)/2],[math.sqrt(2)/2,math.sqrt(2)/2]]
bottomleft = [[-math.sqrt(2)/2, -math.sqrt(2)/2],[-math.sqrt(2)/2,math.sqrt(2)/2]]
bottomright = [[math.sqrt(2)/2, -math.sqrt(2)/2],[-math.sqrt(2)/2,-math.sqrt(2)/2]]



#https://i.imgur.com/O63B9j5.png

x, y = ellipse(x_center=x_center, y_center=y_center, 
                   ax1 = bottomleft,
                   a=2, b =1)
fig.add_scatter(
            x=x,
            y=y,
            mode = 'lines',
            fill="toself",
            fillcolor= "White",
            line=dict(color="Black")
            )

fig.update_xaxes(range=[0, 20], showgrid=False)
fig.update_yaxes(range=[0, 20])

#fig.update_layout(width =600, height=525)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15251231527093595, 'fraction_numerical': 0.04123152709359606, 'mean_word_length': 3.4963455149501663, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '293176', 'license': 'Apache-2.0', 'n_tokens_mistral': 8007, 'n_tokens_neox': 7440, 'n_words': 1463, 'path': 'SOM template/.ipynb_checkpoints/som_vis_janek-checkpoint.ipynb', 'repo_name': 'flurputzer/SOS_MetroMaps', 'size': '20306'}