<jupyter_start><jupyter_text>Exploring conversion from distance matrix to probabilities of changing the hidden state in Forward-Backward algorithm<jupyter_code>import pandas as pd
import numpy as np

from copy import deepcopy
df = pd.read_csv('../configs/matrix.csv', sep=';', index_col=0)
sorted_header = list(sorted(list(df)))
df = df.sort_index()[sorted_header]
print(np.around(df.values, 3))<jupyter_output>[[0.    0.068 0.061 0.007 0.082 0.084 0.069 0.033 0.084]
 [0.068 0.    0.046 0.068 0.098 0.113 0.085 0.071 0.113]
 [0.061 0.046 0.    0.061 0.091 0.106 0.077 0.064 0.106]
 [0.007 0.068 0.061 0.    0.082 0.084 0.068 0.032 0.084]
 [0.082 0.098 0.091 0.082 0.    0.127 0.09  0.085 0.127]
 [0.084 0.113 0.106 0.084 0.127 0.    0.114 0.087 0.06 ]
 [0.069 0.085 0.077 0.068 0.09  0.114 0.    0.072 0.114]
 [0.033 0.071 0.064 0.032 0.085 0.087 0.072 0.    0.087]
 [0.084 0.113 0.106 0.084 0.127 0.06  0.114 0.087 0.   ]]<jupyter_text>The aim is to convert distances between populations into probabilities<jupyter_code>def probability_gen(df, scale=3):
    for i, pop in enumerate(list(df)):
        row = deepcopy(df[pop].values)
        # set distance between population and itself from 0 to (1/3 * minimum)
        row[i] = np.min(row[np.nonzero(row)]) / scale
        s = np.log(row)
        s = s / np.sum(s)
        yield np.around(s, 3)
g = probability_gen(df)
matrix = np.vstack(tuple(g))
matrix
state_matrix = np.array(
    [[0.2  , 0.09 , 0.094, 0.164, 0.084, 0.083, 0.089, 0.114, 0.083],
     [0.11 , 0.171, 0.126, 0.11 , 0.095, 0.089, 0.101, 0.108, 0.089],
     [0.112, 0.123, 0.166, 0.112, 0.096, 0.09 , 0.102, 0.11 , 0.09 ],
     [0.164, 0.09 , 0.094, 0.2  , 0.084, 0.083, 0.089, 0.114, 0.083],
     [0.112, 0.104, 0.107, 0.112, 0.161, 0.092, 0.108, 0.11 , 0.092],
     [0.109, 0.096, 0.099, 0.109, 0.091, 0.172, 0.095, 0.107, 0.123],
     [0.114, 0.105, 0.109, 0.114, 0.102, 0.092, 0.16 , 0.112, 0.092],
     [0.128, 0.099, 0.103, 0.128, 0.092, 0.091, 0.099, 0.169, 0.091],
     [0.109, 0.096, 0.099, 0.109, 0.091, 0.123, 0.095, 0.107, 0.172]]
)
state_matrix[2, 3]
import pathlib
p = pathlib.Path('../configs/matrix.csv')
print(p.parent)
p.name<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18932481751824817, 'fraction_numerical': 0.27965328467153283, 'mean_word_length': 2.606907894736842, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '400916', 'license': 'MIT', 'n_tokens_mistral': 1417, 'n_tokens_neox': 984, 'n_words': 253, 'path': 'notebooks/distances2probabilities.ipynb', 'repo_name': 'smetam/pylae', 'size': '2192'}