<jupyter_start><jupyter_text>Machine Intelligence II - Team MensaNord Sheet 08 - <NAME>- <NAME>- <NAME>- <NAME>- <NAME><jupyter_code>from __future__ import division, print_function
import matplotlib.pyplot as plt
%matplotlib inline
import scipy.stats
import numpy as np<jupyter_output><empty_output><jupyter_text>Exercise 1<jupyter_code>def E(W, s):
    N = len(s)
    return -0.5 * np.sum(W[i, j] * s[i] * s[j] for i, j in np.ndindex(N, N))
N = 6
beta_0 = 0.007
tau = 1.06
epsilon = 1e-20
t_max = 150
W = np.random.random(size=(N, N))
W = (W + W.T) / 2  # make symmetric
for i in range(N):
    W[i, i] = 0
    
plt.imshow(W)<jupyter_output><empty_output><jupyter_text>Simulation with M=1<jupyter_code>M = 1
beta = beta_0
s = np.random.choice([-1, 1], N)
temperatures = np.zeros(t_max)
energies = np.zeros(t_max)
%%time
validation_min = E(W, s)

for t in range(t_max):
    for m in range(M):        
        i = np.random.randint(0, 6)
        s_local = np.copy(s)
        s_local[i] *= -1
        E_1 = E(W, s)
        E_2 = E(W, s_local)
        E_d = E_2 - E_1
        P = 1 / (1 + np.exp(beta*E_d))
        # print("\nt:", t, " i:", i, "\n s1:", s, "\tE1:", E_1, "\n s2:", s_local, "\tE2:", E_2)
        if np.random.random() < P:
            s = np.copy(s_local)
            # print("new s")
        if E(W, s) < validation_min:
            validation_min = E(W, s)
            
    temperatures[t] = 1 / beta
    energies[t] = E(W, s)
    beta *= tau
plt.figure(figsize=(10, 5))
plt.plot(temperatures)
plt.xlabel('t')
plt.ylabel('Temperature')
plt.figure(figsize=(10, 5))
plt.plot(energies, '.-')
plt.xlabel('t')
plt.ylabel('Energy')
s<jupyter_output><empty_output><jupyter_text>Simulation with M=500<jupyter_code>M = 500
beta = beta_0
s = np.random.choice([-1, 1], N)
temperatures = np.zeros(t_max)
energies = np.zeros(t_max)
%%time

validation_min = E(W, s)

for t in range(t_max):
    for m in range(M):        
        i = np.random.randint(0, 6)
        s_local = np.copy(s)
        s_local[i] *= -1
        E_1 = E(W, s)
        E_2 = E(W, s_local)
        E_d = E_2 - E_1
        P = 1 / (1 + np.exp(beta*E_d))
        # print("\nt:", t, " i:", i, "\n s1:", s, "\tE1:", E_1, "\n s2:", s_local, "\tE2:", E_2)
        if np.random.random() < P:
            s = np.copy(s_local)
            # print("new s")
        if E(W, s) < validation_min:
            validation_min = E(W, s)
            
    temperatures[t] = 1 / beta
    energies[t] = E(W, s)
    beta *= tau
plt.figure(figsize=(10, 5))
plt.plot(temperatures)
plt.xlabel('t')
plt.ylabel('Temperature')
plt.figure(figsize=(10, 5))
plt.plot(energies, '.-')
plt.xlabel('t')
plt.ylabel('Energy')
s<jupyter_output><empty_output><jupyter_text>All possible states<jupyter_code># generate all posible states & energies
all_states = [[0, 0, 0, 0, 0, 0] for i in range(2**6)]
all_energies = [0.0 for i in range(2**6)]
for si in range(2**6):
    all_states[si] = [int(x) for x in list('{0:06b}'.format(si))]
    all_energies[si] = E(W, all_states[si])

plt.figure(figsize=(10, 5))
plt.scatter(range(2**6), all_energies)
plt.title('histogram of all possible energies')
plt.grid()
plt.show()

probab_beta = [0.005, 1, 3]

for beta in probab_beta:
    Z = 0
    for en in all_energies:
        Z += np.exp(-beta * en)
    
    all_probabilities = [0.0 for i in range(2**6)]
    for si in range(2**6):
        all_probabilities[si] = np.exp(-beta * all_energies[si])
    
    plt.figure(figsize=(10, 5))
    plt.scatter(range(2**6), all_probabilities)
    plt.title('histogram of all possible probabilities for beta {}'.format(beta))
    plt.grid()
    plt.show()<jupyter_output><empty_output><jupyter_text>Exercise 2<jupyter_code># Other parameters and W from exercise 1.
epsilon = 1e-50
s = np.random.choice([-1., 1.], N)
e = np.zeros_like(s)
beta = beta_0
temperatures = np.zeros(t_max)
energies = np.zeros(t_max)
%%time

for t in range(t_max):
    #print('t =', t, '- beta =', beta)
    
    distance = np.inf
    while distance >= epsilon:
        e_old = e.copy()
        for i in range(N):
            neighbors = range(N)
            neighbors.remove(i)
            e[i] = -np.sum(W[i, j] * s[j] for j in neighbors)
            s[i] = np.tanh(-beta * e[i])
        #print(distance)
        distance = np.linalg.norm(e - e_old)
        
    temperatures[t] = 1 / beta
    energies[t] = E(W, s)
        
    beta *= tau
    #print('-'*10)
plt.figure(figsize=(10, 5))
plt.plot(temperatures)
plt.xlabel('t')
plt.ylabel('Temperature')
plt.figure(figsize=(10, 5))
plt.plot(energies, '.-')
plt.xlabel('t')
plt.ylabel('Energy')
s<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17602427921092564, 'fraction_numerical': 0.02969867765011923, 'mean_word_length': 2.8354114713216956, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '490836', 'license': 'MIT', 'n_tokens_mistral': 2009, 'n_tokens_neox': 1918, 'n_words': 428, 'path': 'sheet08/sheet08.ipynb', 'repo_name': 'jrieke/machine-intelligence-2', 'size': '4653'}