<jupyter_start><jupyter_text>Self Organising Associative Memory<jupyter_code>import os
from dotenv import load_dotenv

load_dotenv()
config = {'db_name': os.getenv("DB_NAME"),
              'db_username': os.getenv("DB_USERNAME"),
              'db_password': os.getenv("DB_PASSWORD"),
              'db_system': os.getenv("DB_SYSTEM"),
              'db_config_file_path': os.getenv("DB_CONFIG_PATH"),
              'db_queries_file_path': os.getenv("DB_QUERIES_PATH"),
              'fabric_graph' : 'HSG',
              'scheduler_address': os.getenv("DASK_SCHEDULER")}<jupyter_output><empty_output><jupyter_text>Libarary functions to visualise data<jupyter_code># library to plot data
#
import plotly.graph_objects as go

# plot the fabric structure
#
def plot_fabric_3d(fabric, title, neurons_to_plot=None, short_term_memory=1):
    x_node = []
    y_node = []
    z_node = []
    colors = []
    x_edge = []
    y_edge = []
    z_edge = []
    labels = []
    sizes = []
    pairs = set()
    nos_bmus = 0
    
    if neurons_to_plot is None:
        neurons_to_plot = [n for n in range(short_term_memory)]
            
    for coord_id in fabric['neuro_columns'].keys():
        
        # if its a bmu
        #
        if fabric['neuro_columns'][coord_id]['n_bmu'] > 0:
            nos_bmus += 1
            
        for neuron_id in neurons_to_plot:
            
            # if there are edges for this neuron_id
            #
            if sum([1 for edge in fabric['neuro_columns'][coord_id]['neuro_column'] if fabric['neuro_columns'][coord_id]['neuro_column'][edge]['neuron_id'] == neuron_id]) > 0:
                
                # if neuron in the top of the column and has been trained then increase size
                #
                if neuron_id == neurons_to_plot[0] and fabric['neuro_columns'][coord_id]['n_bmu'] + fabric['neuro_columns'][coord_id]['n_nn'] > 0:
                    sizes.append(20 + fabric['neuro_columns'][coord_id]['n_bmu'])
                else:
                    sizes.append(15)

                nc = fabric['neuro_columns'][coord_id]['neuro_column']

                x = fabric['neuro_columns'][coord_id]['coord'][0]
                y = fabric['neuro_columns'][coord_id]['coord'][1]
                z = max(neurons_to_plot) - neuron_id

                x_node.append(x)
                y_node.append(y)
                z_node.append(z)

                colors.append(fabric['neuro_columns'][coord_id]['community'])

                txt = 'NeuroColumn: {}<br>Neuron: {}<br>n_BMU: {}<br>last_bmu: {}<br>n_nn: {}<br>last_nn: {}<br>mean_dist: {:.2f}<br>mean_sim: {:.2f}<br>community:{}'.format(coord_id, 
                                                                                                                           neuron_id, 
                                                                                                                           fabric['neuro_columns'][coord_id]['n_bmu'],
                                                                                                                           fabric['neuro_columns'][coord_id]['last_bmu'],
                                                                                                                           fabric['neuro_columns'][coord_id]['n_nn'],
                                                                                                                           fabric['neuro_columns'][coord_id]['last_nn'],
                                                                                                                           fabric['neuro_columns'][coord_id]['mean_distance'] if 'mean_distance' in fabric['neuro_columns'][coord_id] else None,
                                                                                                                           fabric['neuro_columns'][coord_id]['mean_similarity'] if 'mean_similarity' in fabric['neuro_columns'][coord_id] else None,
                                                                                                                           fabric['neuro_columns'][coord_id]['community'],
                                                                                                                           )
                edges = list(fabric['neuro_columns'][coord_id]['neuro_column'].keys())
                edges.sort()
                for edge in edges:
                    if fabric['neuro_columns'][coord_id]['neuro_column'][edge]['neuron_id'] == neuron_id:
                        
                        # get the numeric if it exists
                        #
                        if 'numeric' in fabric['neuro_columns'][coord_id]['neuro_column'][edge]:
                            numeric = '{:.2f}'.format(fabric['neuro_columns'][coord_id]['neuro_column'][edge]['numeric'])
                        else:
                            numeric = None
                        
                        txt = '{}<br>{}: {} {}: {} Prob: {:.2f} Numeric: {}'.format(txt, 
                                                                              fabric['neuro_columns'][coord_id]['neuro_column'][edge]['edge_type'],
                                                                              fabric['neuro_columns'][coord_id]['neuro_column'][edge]['edge_uid'],
                                                                              fabric['neuro_columns'][coord_id]['neuro_column'][edge]['target_type'],
                                                                              fabric['neuro_columns'][coord_id]['neuro_column'][edge]['target_uid'],
                                                                              fabric['neuro_columns'][coord_id]['neuro_column'][edge]['prob'],
                                                                              numeric
                                                                             )
                labels.append(txt)
                
                # connect neurons in same column
                #
                if neuron_id < neurons_to_plot[-1]:
                    pair = (min((x, y, neuron_id), (x, y, neuron_id + 1)), max((x, y, neuron_id), (x, y, neuron_id + 1)))

                    pairs.add(pair)

                    x_edge.append(x)
                    x_edge.append(x)
                    x_edge.append(None)

                    y_edge.append(y)
                    y_edge.append(y)
                    y_edge.append(None)

                    nn_z = max(neurons_to_plot) - neuron_id - 1
                    z_edge.append(z)
                    z_edge.append(nn_z)
                    z_edge.append(None)

                # connect neuro_columns
                #
                for nn_id in fabric['neuro_columns'][coord_id]['nn']:
                    nn_x = fabric['neuro_columns'][nn_id]['coord'][0]
                    nn_y = fabric['neuro_columns'][nn_id]['coord'][1]

                    pair = (min((x, y, neuron_id), (nn_x, nn_y, neuron_id)), max((x, y, neuron_id), (nn_x, nn_y, neuron_id)))

                    if pair not in pairs:
                        pairs.add(pair)
                        nn_z = max(neurons_to_plot) - neuron_id
                        x_edge.append(x)
                        x_edge.append(nn_x)
                        x_edge.append(None)

                        y_edge.append(y)
                        y_edge.append(nn_y)
                        y_edge.append(None)

                        z_edge.append(z)
                        z_edge.append(nn_z)
                        z_edge.append(None)

    neuron_scatter = go.Scatter3d(x=x_node, y=y_node, z=z_node, hovertext=labels, mode='markers', marker=dict(size=sizes, color=colors, opacity=1.0))

    amf_edge_scatter = go.Scatter3d(x=x_edge, y=y_edge, z=z_edge, mode='lines', line=dict(width=0.5, color='grey'))

    fig = go.Figure(data=[neuron_scatter, amf_edge_scatter])
    fig.update_layout(width=1200, height=1200, title=dict(text=title),
                      scene=dict(xaxis_title='X Coord', yaxis_title='Y Coord', zaxis_title='Neuron ID'))
    print('Nos Neuron_columns: {} Nos BMUs: {}, Nos Mapped: {} Mean Distance: {:.2f} Mean similarity: {:.2f}'.format(len(fabric['neuro_columns']),
                                                                                                                    nos_bmus,
                                                                                                                    fabric['mapped'],
                                                                                                                    fabric['mean_distance'],
                                                                                                                    fabric['mean_similarity']))
    
    fig.show()

def explain_search_por(search_por):
    
    total_dist = sum([search_por[neuron]['distance'] for neuron in search_por])
    neuron_dist = [[neuron, search_por[neuron]['distance']] for neuron in search_por]
    neuron_dist.sort(key=lambda x: x[1], reverse=True)
    
    print('Total Distance: {:.2f} Distance contribution by neuron:'.format(total_dist))
    for n in neuron_dist:
        print('  Neuron: {} Distance: {:.2f} %Distance: {:.2%}'.format(n[0], n[1], n[1] / total_dist))
    
    for neuron in search_por:
        total_dist = search_por[neuron]['distance']
        edge_dist = [(edge, search_por[neuron]['edges'][edge]['distance']) for edge in search_por[neuron]['edges']]
        print('')
        edge_dist.sort(key=lambda x: x[1], reverse=True)
        for edge in edge_dist:
            print('      Edge: {} Distance: {:.2f} %Distance: {:.2%}'.format(edge[0], edge[1], edge[1] / total_dist))
            edge_data = search_por[neuron]['edges'][edge[0]]
            print('        Prob distance: {:.2f} neuro_column: {:.2f} sdr: {:.2f}'.format(edge_data['prob']['distance'],edge_data['prob']['nc'],edge_data['prob']['compare_nc']))
            if 'numeric' in edge_data:
                print('        Numeric distance: {:.2f} neuro_column: {:.2f} sdr: {:.2f}'.format(edge_data['numeric']['distance'],edge_data['numeric']['nc'],edge_data['prob']['compare_nc']))
            print('')
        
    return neuron_dist

def explain_anomaly_por(anomaly_por):
    anomalies = [(ref_id, anomaly_por[ref_id]['update_id']) for ref_id in anomaly_por]
    anomalies.sort(key=lambda x: x[1])
    for anomaly in anomalies:
        print('Anomaly on update: {} ref_id: {} occurred on Neuro_Column: {} Distance: {:.2f} Threshold:{:.2f}'.format(anomaly[1], anomaly[0], 
                                                                                                                      anomaly_por[anomaly[0]]['bmu_coord'],
                                                                                                                      anomaly_por[anomaly[0]]['distance'], 
                                                                                                                      anomaly_por[anomaly[0]]['threshold']))
        
        explain_search_por(anomaly_por[anomaly[0]]['por'])

# plot matrix profile
#
def plot_matrix_profile(amf, por_results, std_factor = 1):
    sp_mp = []
    sp_anomaly = []
    sp_motif = []
    sp_anomaly_threshold = []
    sp_motif_threshold = []
    sp_mean = []
    sp_std_upper = []
    sp_std_lower = []

    sp_anomalies = amf.get_anomalies()
    sp_motifs = amf.get_motifs()

    for por in por_results:
        sp_mp.append(por['bmu_distance'])

        sp_mean.append(por['mean_distance'])
        sp_std_upper.append(por['mean_distance'] + (std_factor * por['std_distance']))
        sp_std_lower.append(max(por['mean_distance'] - (std_factor * por['std_distance']), 0.0) )


        sp_anomaly_threshold.append(por['anomaly_threshold'])
        sp_motif_threshold.append(por['motif_threshold'])

        if por['ref_id'] in sp_anomalies:
            sp_anomaly.append(por['bmu_distance'])
        else:
            sp_anomaly.append(None)
        if por['ref_id'] in sp_motifs:
            sp_motif.append(por['bmu_distance'])
        else:
            sp_motif.append(None)

    
    sp_mp_scatter = go.Scatter(name='Matrix Profile', x=[idx for idx in range(len(sp_mp))], y=sp_mp, mode='lines', line=dict(width=2.0, color='green'))
    
    sp_anomaly_scatter = go.Scatter(name='anomalies', x=[idx for idx in range(len(sp_anomaly))], y=sp_anomaly, mode='markers+text', marker=dict(size=10, color='red', opacity=0.7))
    sp_motif_scatter = go.Scatter(name='motifs', x=[idx for idx in range(len(sp_motif))], y=sp_motif, mode='markers+text', marker=dict(size=10, color='green', opacity=0.7))

    sp_motif_threshold_scatter = go.Scatter(name='motif threshold', x=[idx for idx in range(len(sp_motif_threshold))], y=sp_motif_threshold, mode='lines', line=dict(width=1.0, color='blue'))
    sp_anomaly_threshold_scatter = go.Scatter(name='anomaly threshold', x=[idx for idx in range(len(sp_anomaly_threshold))], y=sp_anomaly_threshold, mode='lines', line=dict(width=1.0, color='red'))

    sp_mean_scatter = go.Scatter(name='mean', x=[idx for idx in range(len(sp_mean))], y=sp_mean, mode='lines', line=dict(width=2.0, color='black'))
    sp_std_upper_scatter = go.Scatter(name='stdev', x=[idx for idx in range(len(sp_std_upper))], y=sp_std_upper, mode='lines', line=dict(width=2.0, color='yellow'))
    sp_std_lower_scatter = go.Scatter(name='stdev', x=[idx for idx in range(len(sp_std_lower))], y=sp_std_lower, mode='lines', line=dict(width=2.0, color='yellow'))
    
    fig = go.Figure(data=[sp_mp_scatter, sp_anomaly_scatter, sp_motif_scatter, sp_motif_threshold_scatter, sp_anomaly_threshold_scatter, sp_mean_scatter, sp_std_upper_scatter, sp_std_lower_scatter,])
    fig.update_layout(scene=dict(xaxis_title='REF ID', yaxis_title='Similarity'), width=1400, height=900,
                      title=dict(text='AMFabric Pooler Anomalies & Motifs'))
    fig.show()<jupyter_output><empty_output><jupyter_text>Instantiate Distributed AMFabric object<jupyter_code>from src.distributed_amfabric import DistributedAMFabric

amf = DistributedAMFabric(config=config)<jupyter_output><empty_output><jupyter_text>Extract fabric Area and plot<jupyter_code>fabric_area = 'DEF_Ltd'

decoded_fabric = amf.decode_fabric(fabric_uid=fabric_area, 
                                   all_details=True, 
                                   min_cluster_size= 1, 
                                   start_threshold= 0.5, 
                                   step= 0.001, 
                                   lock_fabric= True)
decoded_fabric=decoded_fabric.result()  

plot_fabric_3d(fabric=decoded_fabric, 
               title=fabric_area, 
               neurons_to_plot=[0], 
               short_term_memory=1)
explain_anomaly_por(decoded_fabric['anomaly'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12270062664240954, 'fraction_numerical': 0.007411899467690856, 'mean_word_length': 1.5834638816362054, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 107, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '361538', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4294, 'n_tokens_neox': 4054, 'n_words': 652, 'path': 'tutorial/soam_viz.ipynb', 'repo_name': 'kontexia/soam', 'size': '14841'}