<jupyter_start><jupyter_text>Set up demo data<jupyter_code>num_samples = 25
data_output_dimension = 100
data_latent_dimension = 3

demo_data = get_random_data(
    num_samples,
    data_output_dimension,
    latent_dim=data_latent_dimension
)<jupyter_output><empty_output><jupyter_text>Set up HAH network<jupyter_code>input_layer_size = data_output_dimension
nodes_per_layer = [2]

hah_network = MultilayerHahNetwork(
    input_layer_size,
    nodes_per_layer,
    has_bias=False,
    act_type='linear',
)<jupyter_output><empty_output><jupyter_text>Train network<jupyter_code>hah_network.train(demo_data, num_epochs=1000)<jupyter_output><empty_output><jupyter_text>Compare network to top PC<jupyter_code># Get top principle component for input sample
U, S, V = np.linalg.svd(np.array(demo_data))
_idx = np.argsort(S)[-nodes_per_layer[0]:]
top_principal_comps = V[_idx, :].T

# Get pattern of input weights in network
hah_input_weights = np.squeeze(hah_network.layers[0].input_weights)

# Show the inner product of top two PCs with learned input weights
inner_prod_mat = top_principal_comps.T.dot(hah_input_weights)
print(inner_prod_mat)<jupyter_output>[[10.7646494  13.02089182]
 [ 0.25147563  3.3972458 ]]<jupyter_text>If everything has gone well, the scatter plot below should show that the learned input-weights for this model correspond to the first principle component for this data<jupyter_code>fig, ax = plt.subplots(nrows=2, ncols=1)

fig.suptitle('Comparison of HAH-learned input weights and principle components loadings')

for pc_idx in range(2):
    
    if np.abs(inner_prod_mat[pc_idx, pc_idx]) > np.abs(inner_prod_mat[pc_idx, 1 - pc_idx]):
        ha_idx = pc_idx
    else:
        ha_idx = 1 - pc_idx

    ax[pc_idx].set_title('PC {}'.format(pc_idx))
    ax[pc_idx].scatter(
        top_principal_comps[:, pc_idx].ravel(),
        hah_input_weights[:, ha_idx].ravel()
    )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10057321521625846, 'fraction_numerical': 0.028139656070870246, 'mean_word_length': 5.7368421052631575, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '657725', 'license': 'MIT', 'n_tokens_mistral': 751, 'n_tokens_neox': 707, 'n_words': 140, 'path': 'demos/Test extraction of top principal components.ipynb', 'repo_name': 'bmcmenamin/hebbnets', 'size': '1919'}