<jupyter_start><jupyter_text>import modules and get command-line parameters if running as script<jupyter_code>from probrnn import models, data, inference
import numpy as np
import json 
from matplotlib import pyplot as plt
from IPython.display import clear_output<jupyter_output><empty_output><jupyter_text>parameters for the model and training<jupyter_code>params = \
    {
        "N_ITERATIONS": 10 ** 5,
        "VALIDATE_EACH": 100,
        "SAVE_EACH": 1000,
        "LOG_EVERY": 50,
        "LEARNING_RATE": 0.0001,
        "N_HIDDEN": 256,
        "N_BINS": 50,
        "BATCH_SIZE": 50,
    }<jupyter_output><empty_output><jupyter_text>Get some Mackey-Glass model data, which evolves according to:$$\frac{dx}{dt} = \frac{\beta \, x_{t - \tau}}{1 + (x_{t - \tau})^n} - \gamma \, x_t$$We take random initial conditions<jupyter_code>datastruct = data.ToyDataMissing(n_bins=params["N_BINS"])

x = datastruct.simulate()

plt.figure()
plt.plot(x)
plt.show()<jupyter_output><empty_output><jupyter_text>do some training with missing values - for this we have to specify the imputing function<jupyter_code>model = models.NADE(datastruct, params=params)

training = models.Training(
                            models.NADE, 
                            "../models/toy_nade_missing", 
                            "../models/toy_nade_missing_training.json",
                            imputer=inference.NaiveSIS(model)
                          )

def print_function(trer, i, batch):
    if i % 10 == 0:
        clear_output()
        print "loss: {}; iteration {}".format(np.mean(trer[-100:]), i)

training.train(print_function)<jupyter_output><empty_output><jupyter_text>visualize the training errors<jupyter_code>with open("../models/toy_nade_missing_training.json") as f:
    errs = json.load(f)

plt.figure()
plt.plot(np.array(errs["training_error"])[:, 0], 
         np.array(errs["training_error"])[:, 1])
plt.plot(np.array(errs["validation_error"])[:, 0], 
         np.array(errs["validation_error"])[:, 1], 'r')
plt.legend(["training", "validation"])
plt.show()<jupyter_output><empty_output><jupyter_text>plot some weight traces<jupyter_code>for x in errs.keys():
    if x != "training_error" and x != "validation_error" and "train" not in x:
        plt.figure()
        for key in errs[x].keys():
            if key == "mean":
                plt.plot(errs[x][key], 'b', linewidth=5.0)
            elif key == "random":
                plt.plot(errs[x][key], 'c')
            else:
                plt.plot(errs[x][key], 'b', linestyle='--')

        plt.title("variable: {}".format(x))
plt.show()<jupyter_output><empty_output><jupyter_text>load trained model if necessary<jupyter_code>load_name = "../models/toy_nade_missing_21000"
model = models.NADE(datastruct, fn=load_name)
print json.dumps(model.params, indent=4)<jupyter_output>restoring...
INFO:tensorflow:Restoring parameters from ../models/toy_nade_missing_21000.ckpt
{
    "N_BINS": 50, 
    "LEARNING_RATE": 0.0001, 
    "BATCH_SIZE": 1, 
    "SAVE_EACH": 1000, 
    "N_ITERATIONS": 1000000, 
    "VALIDATE_EACH": 100, 
    "LOG_EVERY": 50, 
    "N_HIDDEN": 128, 
    "N_PARTICLES": 64
}<jupyter_text>try some sampling<jupyter_code>x = model.sample(200)

plt.plot(x)
plt.show()<jupyter_output><empty_output><jupyter_text>try some imputation<jupyter_code>x = datastruct.simulate()
x_missing = np.copy(x)
x_missing[np.random.choice(len(x), replace=False, size=125)] = np.nan

estimate = inference.NaiveSIS(model, x_missing, 1000, binned=False, quiet=False).estimate()

plt.figure()
plt.plot(x_missing, "rx")
plt.plot(x, 'r')
plt.plot(estimate)
plt.legend(["observed", "ground truth", "estimate"])
plt.show()<jupyter_output>100%|██████████| 150/150 [00:05<00:00, 27.57it/s]
{'fraction_non_alphanumeric': 0.1545041432771986, 'fraction_numerical': 0.028869286287089013, 'mean_word_length': 3.665835411471322, 'pattern_counts': {'":': 19, '<': 36, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '588062', 'license': 'MIT', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1341, 'n_words': 245, 'path': 'examples/mackey-class.ipynb', 'repo_name': 'zalandoresearch/probrnn', 'size': '3741'}