<jupyter_start><jupyter_text>Load data<jupyter_code>cifar_train, cifar_test, cifar_mean = get_cifar10_data("../data/")

# Create noisy examples (add gaussian noise)
noise_multiplier = 0.05
noise_train = cifar_train + np.random.normal(size=cifar_train.shape) * noise_multiplier
noise_train /= (np.linalg.norm(noise_train, axis=1) / np.linalg.norm(cifar_train, axis=1))[:, np.newaxis]

noise_test = cifar_test + np.random.normal(size=cifar_test.shape) * noise_multiplier
noise_test /= (np.linalg.norm(noise_test, axis=1) / np.linalg.norm(cifar_test, axis=1))[:, np.newaxis]
fig, axs = plt.subplots(1,3, figsize=(15, 15))
axs[0].imshow(cifar_mean.reshape((28 ,28)), cmap='gray', vmin=0, vmax=1)
axs[0].set_title("pixel wise mean")
axs[1].imshow((cifar_mean + cifar_train[0]).reshape((28 ,28)), cmap='gray', vmin=0, vmax=1)
axs[1].set_title("training sample (with added mean)")
axs[2].imshow((cifar_mean + noise_train[0]).reshape((28 ,28)), cmap='gray', vmin=0, vmax=1)
axs[2].set_title("noise sample (with added mean)")<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>model, losses = train_cnce(batch_size=1000, cifar10_data_path="../data/", noise_multiplier=0.05,
                           max_steps=10000)
plt.plot(losses)
plt.yscale("log")
plt.xlabel("steps")
plt.ylabel("negative cNCE loss")<jupyter_output><empty_output><jupyter_text>Inspect and evaluate<jupyter_code>def plot_filters(V):
    num_filters = V.shape[0]
    num_columns = 8
    num_rows = num_filters // num_columns
    fig, axs = plt.subplots(num_rows, num_columns, figsize=(15, 15))
    for k in range(num_filters):
        axs[k // num_columns, k % num_columns].imshow(V[k].reshape((28 ,28)), cmap='gray')
        
def evaluate(samples):
    log_probs = model(torch.tensor(samples, device=device)).cpu().detach().numpy()
    print(f"Mean log_p_theta(x): {log_probs.mean()}")
    print(f"Std log_p_theta(x): {log_probs.std()}")
plot_filters(model.V.cpu().detach().numpy())
evaluate(cifar_train)
evaluate(cifar_test)
evaluate(noise_train)
mnist_train, mnist_test, mnist_mean = get_mnist_data("../data")
evaluate(mnist_train)<jupyter_output>Mean log_p_theta(x): 112.49588237496752
Std log_p_theta(x): 18.885967675256396
{'fraction_non_alphanumeric': 0.16772727272727272, 'fraction_numerical': 0.045, 'mean_word_length': 6.312292358803987, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '209375', 'license': 'Apache-2.0', 'n_tokens_mistral': 965, 'n_tokens_neox': 889, 'n_words': 130, 'path': 'src/Exercise 2 - cNCE.ipynb', 'repo_name': 'TobiasNorlund/learning-feature-representations', 'size': '2200'}