<jupyter_start><jupyter_text>Evaluate OOD<jupyter_code>model = configuration.setup_model(results['config'])
model.load_state_dict(torch.load(os.path.join(result_dir, "best_model_" + str(exp_id) + ".pth.tar")))
model
def evaluate_along_var(i):
    x = []
    y = []
    set_inter_node(i)
    for j in np.arange(-20, 20, 0.5):
        set_inter_shift(j)
        set_seed(results['seed'])
        # dataloader
        valid_loader = configuration.setup_dataloader(results['config'], 'valid')
        valid_loss, valid_losses = evaluate_epoch(model=model, device='cpu', data_loader=valid_loader, epoch=0)
        x.append(j)
        y.append(np.mean([valid_losses[j]['recon_loss'].numpy() for j in range(len(valid_losses))]))
    return x, y
x0, y0 = evaluate_along_var(0)
x1, y1 = evaluate_along_var(1)
plt.figure(dpi=200)
plt.plot(x0, y0, label="direction 0")
plt.title("Out of Distribution Reconstruction loss")
plt.xlabel('Shift along direction at test time')
plt.ylabel('Test reconstruction loss')
plt.legend()
# plt.yscale("log")
# plt.show()
plt.plot(x1, y1, label="direction 1")
# plt.title("Out of Distribution Reconstruction loss")
# plt.xlabel('Shift along direction at test time')
# plt.ylabel('Valid reconstruction loss')
plt.legend()
plt.yscale("log")
plt.show()
ground_truth_matrix = get_ground_truth_matrix()
model_matrix = get_model_matrix()
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(6, 6), dpi=200)
ax1.matshow(model_matrix, cmap="PiYG", vmin=-1.,vmax=1.)
ax1.set_title("model matrix")
ax2.matshow(ground_truth_matrix, cmap="PiYG", vmin=-1.,vmax=1.)
ax2.set_title("ground truth matrix")
plt.show()<jupyter_output><empty_output><jupyter_text>Difference between ground truth and learnt matrices<jupyter_code>def cosine_dist(A, B):
    """
    Compute the sum of cosine distances between the rows of A and B
    """
    dist = 0
    for i in range(len(get_ground_truth_matrix())):
        dist += cosine(A[i], B[i])
    return dist
def cosine_dist_ith_components(A, B, i):
    return cosine(A[i], B[i])
dist_to_ground_truth = []
for theta in np.arange(0, 2*np.pi, 0.01):
    rot = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
    rotated_weights = get_model_matrix()[:, :2].dot(rot)
    dist_to_ground_truth.append(cosine_dist(rotated_weights, get_ground_truth_matrix()))

plt.axvline(x=100*np.pi, ymin=0, ymax=20)
plt.axvline(x=100*1/2*np.pi, ymin=0, ymax=20)
plt.axvline(x=100*3/2*np.pi, ymin=0, ymax=20)
plt.plot(dist_to_ground_truth)
plt.yscale("log")
plt.show()
dist_to_ground_truth = []
for theta in np.arange(0, 2*np.pi, 0.01):
    rot = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
    rotated_weights = get_model_matrix_column_opp()[:, :2].dot(rot)
    dist_to_ground_truth.append(cosine_dist(rotated_weights, get_ground_truth_matrix()))

plt.axvline(x=100*np.pi, ymin=0, ymax=20)
plt.axvline(x=100*1/2*np.pi, ymin=0, ymax=20)
plt.axvline(x=100*3/2*np.pi, ymin=0, ymax=20)
plt.plot(dist_to_ground_truth)
plt.yscale("log")
plt.show()
f,ax=plt.subplots(dpi=200)
for j in range(10):
    dist_to_ground_truth = []
    for theta in np.arange(0, 2*np.pi, 0.01):
        rot = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
        rotated_weights = get_model_matrix()[:, :2].dot(rot)
        dist_to_ground_truth.append(cosine_dist_ith_components(rotated_weights, get_ground_truth_matrix(),j))
    ax.plot([0.01/(np.pi)*i for i in range(len(dist_to_ground_truth))], dist_to_ground_truth, label=str(j))
ax.legend()
ax.xaxis.set_major_formatter(tck.FormatStrFormatter('%g $\pi$'))
ax.xaxis.set_major_locator(tck.MultipleLocator(base=0.5))
ax.axvline(x=0, ymin=0, ymax=20)
ax.axvline(x=0.5, ymin=0, ymax=20)
ax.axvline(x=1, ymin=0, ymax=20)
ax.axvline(x=3/2, ymin=0, ymax=20)
ax.axvline(x=2, ymin=0, ymax=20)
ax.set_xlabel(r'rotation of angle $\theta$')
ax.set_ylabel('cosine distance between ith lines')
ax.set_title('Distances between lines of inferred and true matrices')
# plt.yscale("log")
plt.show()
f,ax=plt.subplots(dpi=200)
for j in range(10):
    dist_to_ground_truth = []
    for theta in np.arange(0, 2*np.pi, 0.01):
        rot = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
        rotated_weights = get_model_matrix_column_opp()[:, :2].dot(rot)
        dist_to_ground_truth.append(cosine_dist_ith_components(rotated_weights, get_ground_truth_matrix(), j))
    ax.plot([0.01/(np.pi)*i for i in range(len(dist_to_ground_truth))], dist_to_ground_truth, label=str(j))
ax.legend()
ax.xaxis.set_major_formatter(tck.FormatStrFormatter('%g $\pi$'))
ax.xaxis.set_major_locator(tck.MultipleLocator(base=0.5))
ax.axvline(x=0, ymin=0, ymax=20)
ax.axvline(x=0.5, ymin=0, ymax=20)
ax.axvline(x=1, ymin=0, ymax=20)
ax.axvline(x=3/2, ymin=0, ymax=20)
ax.axvline(x=2, ymin=0, ymax=20)
# plt.yscale("log")
ax.set_title('Distances between lines of inferred and true matrices')
ax.set_xlabel(r'rotation of angle $\theta$')
ax.set_ylabel('cosine distance between ith lines')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17380103051922316, 'fraction_numerical': 0.03408640507332541, 'mean_word_length': 5.649538866930171, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '434219', 'license': 'MIT', 'n_tokens_mistral': 2193, 'n_tokens_neox': 2071, 'n_words': 345, 'path': 'notebooks/ResultExploration/Extrapolation.ipynb', 'repo_name': 'Bertinus/causal_cell_embedding', 'size': '5046'}