<jupyter_start><jupyter_text>Licensed under the Apache License, Version 2.0.<jupyter_code>import tensorflow as tf
import os
import itertools

from experimental.attentive_uncertainty.contextual_bandits.pretrain_gnp import train  # local file import
savedir = '/tmp/tfbind8/models/'
context_dim = 2
num_actions = 5
num_target = 50
num_context = 512
data_hparams = tf.contrib.training.HParams(context_dim=context_dim,
                                           num_actions=num_actions,
                                           num_target=num_target,
                                           num_context=num_context)
model_type = 'anp'
x_y_encoder_net_sizes = [HIDDEN_SIZE]*3
global_latent_net_sizes = [HIDDEN_SIZE]*2
local_latent_net_sizes = None
x_encoder_net_sizes = None
HIDDEN_SIZE = 64
decoder_net_sizes = [HIDDEN_SIZE]*3 + [2*num_actions]
heteroskedastic_net_sizes = None
att_type = 'multihead'
att_heads = 8
data_uncertainty = False
beta = 1
temp = 1
betas = [0., 0.5, 1., 2., 5., 1000.]
temps = [1e-3, 0.5, 1., 2., 5., 1000.]
for beta, temp in itertools.product(betas, temps):
  print('beta', beta, 'temperature', temp, flush=True)
  model_hparams = tf.contrib.training.HParams(activation=tf.nn.relu,
                                            output_activation=tf.nn.relu,
                                            x_encoder_net_sizes=x_encoder_net_sizes,
                                            x_y_encoder_net_sizes=x_y_encoder_net_sizes,
                                            global_latent_net_sizes=global_latent_net_sizes,
                                            local_latent_net_sizes=local_latent_net_sizes,
                                            decoder_net_sizes=decoder_net_sizes, 
                                            heteroskedastic_net_sizes=heteroskedastic_net_sizes,
                                            att_type=att_type,
                                            att_heads=att_heads,
                                            model_type=model_type,
                                            data_uncertainty=data_uncertainty,
                                            beta=beta,
                                            temperature=temp)
  save_path = os.path.join(savedir, 'gnp_' + model_type + '_beta_' + str(beta) + '_temp_'+ str(temp) + '.ckpt')
  training_hparams = tf.contrib.training.HParams(lr=0.01,
                                                optimizer=tf.train.RMSPropOptimizer,
                                                num_iterations=10000,
                                                batch_size=10,
                                                num_context=num_context,
                                                num_target=num_target, 
                                                print_every=100,
                                                save_path=save_path,
                                                max_grad_norm=10.0,
                                                is_nll=True)

  train(data_hparams,
        model_hparams,
        training_hparams)<jupyter_output>beta 0.0 temperature 0.001
it: 0, train recon loss: 6387034.0, local kl: 0.0 global kl: 0.0 valid reconstr loss: 44108340.0
Saving best model with reconstruction loss 44108340.0
it: 100, train recon loss: 3.799004554748535, local kl: 0.0 global kl: 0.0 valid reconstr loss: 4.032503604888916
Saving best model with reconstruction loss 4.0325036
it: 200, train recon loss: 3.2728283405303955, local kl: 0.0 global kl: 0.0 valid reconstr loss: 3.3462297916412354
Saving best model with reconstruction loss 3.3462298
it: 300, train recon loss: 2.59476900100708, local kl: 0.0 global kl: 0.0 valid reconstr loss: 2.627716302871704
Saving best model with reconstruction loss 2.6277163
it: 400, train recon loss: 2.2547805309295654, local kl: 0.0 global kl: 0.0 valid reconstr loss: 2.3351688385009766
Saving best model with reconstruction loss 2.3351688
it: 500, train recon loss: 2.110807418823242, local kl: 0.0 global kl: 0.0 valid reconstr loss: 5.039649963378906
it: 600, train recon loss: 7.90053415[...]<jupyter_text>CNP<jupyter_code>model_type = 'cnp'
x_y_encoder_net_sizes = [HIDDEN_SIZE]*4
global_latent_net_sizes = None
local_latent_net_sizes = None

model_hparams = tf.contrib.training.HParams(activation=tf.nn.relu,
                                            output_activation=tf.nn.relu,
                                            x_encoder_net_sizes=x_encoder_net_sizes,
                                            x_y_encoder_net_sizes=x_y_encoder_net_sizes,
                                            global_latent_net_sizes=global_latent_net_sizes,
                                            local_latent_net_sizes=local_latent_net_sizes,
                                            decoder_net_sizes=decoder_net_sizes, 
                                            heteroskedastic_net_sizes=heteroskedastic_net_sizes,
                                            att_type=att_type,
                                            att_heads=att_heads,
                                            model_type=model_type,
                                            data_uncertainty=data_uncertainty)
save_path = os.path.join(savedir, 'gnp_' + model_type + '.ckpt')
training_hparams = tf.contrib.training.HParams(lr=0.01,
                                               optimizer=tf.train.RMSPropOptimizer,
                                               num_iterations=10000,
                                               batch_size=10,
                                               num_context=num_context,
                                               num_target=num_target, 
                                               print_every=50,
                                               save_path=save_path,
                                               max_grad_norm=1000.0)

train(data_hparams,
      model_hparams,
      training_hparams)
model_type = 'cnp'
x_y_encoder_net_sizes = [HIDDEN_SIZE]*4
global_latent_net_sizes = None
local_latent_net_sizes = None

model_hparams = tf.contrib.training.HParams(activation=tf.nn.relu,
                                            output_activation=tf.nn.relu,
                                            x_encoder_net_sizes=x_encoder_net_sizes,
                                            x_y_encoder_net_sizes=x_y_encoder_net_sizes,
                                            global_latent_net_sizes=global_latent_net_sizes,
                                            local_latent_net_sizes=local_latent_net_sizes,
                                            decoder_net_sizes=decoder_net_sizes, 
                                            heteroskedastic_net_sizes=heteroskedastic_net_sizes,
                                            att_type=att_type,
                                            att_heads=att_heads,
                                            model_type=model_type,
                                            data_uncertainty=data_uncertainty)
save_path = os.path.join(savedir, 'gnp_nll_' + model_type + '.ckpt')
training_hparams = tf.contrib.training.HParams(lr=0.01,
                                               optimizer=tf.train.RMSPropOptimizer,
                                               num_iterations=10000,
                                               batch_size=10,
                                               num_context=num_context,
                                               num_target=num_target, 
                                               print_every=50,
                                               save_path=save_path,
                                               max_grad_norm=1000.0,
                                               is_nll=True)

train(data_hparams,
      model_hparams,
      training_hparams)<jupyter_output><empty_output><jupyter_text>NP<jupyter_code>model_type = 'np'
x_y_encoder_net_sizes = [HIDDEN_SIZE]*2
global_latent_net_sizes = [HIDDEN_SIZE]*2
local_latent_net_sizes = None

model_hparams = tf.contrib.training.HParams(activation=tf.nn.relu,
                                            output_activation=tf.nn.relu,
                                            x_encoder_net_sizes=x_encoder_net_sizes,
                                            x_y_encoder_net_sizes=x_y_encoder_net_sizes,
                                            global_latent_net_sizes=global_latent_net_sizes,
                                            local_latent_net_sizes=local_latent_net_sizes,
                                            decoder_net_sizes=decoder_net_sizes, 
                                            heteroskedastic_net_sizes=heteroskedastic_net_sizes,
                                            att_type=att_type,
                                            att_heads=att_heads,
                                            model_type=model_type,
                                            data_uncertainty=data_uncertainty)
save_path = os.path.join(savedir, 'gnp_' + model_type + '.ckpt')
training_hparams = tf.contrib.training.HParams(lr=0.01,
                                               optimizer=tf.train.RMSPropOptimizer,
                                               num_iterations=10000,
                                               batch_size=10,
                                               num_context=num_context,
                                               num_target=num_target, 
                                               print_every=50,
                                               save_path=save_path,
                                               max_grad_norm=1000.0)

train(data_hparams,
      model_hparams,
      training_hparams)<jupyter_output>it: 0, train mse: 65.76097869873047, local kl: 0.0 global kl: 0.054639000445604324 valid mse: 83.96866607666016, local kl: 0.0 global kl: 0.07167519629001617
Saving best model with MSE 83.968666
it: 50, train mse: 57.198280334472656, local kl: 0.0 global kl: 0.017055261880159378 valid mse: 67.92539978027344, local kl: 0.0 global kl: 0.03785083070397377
Saving best model with MSE 67.9254
it: 100, train mse: 30.777847290039062, local kl: 0.0 global kl: 0.03610922396183014 valid mse: 33.16203689575195, local kl: 0.0 global kl: 0.11787480115890503
Saving best model with MSE 33.162037
it: 150, train mse: 24.821523666381836, local kl: 0.0 global kl: 0.04832617938518524 valid mse: 24.692289352416992, local kl: 0.0 global kl: 0.058483920991420746
Saving best model with MSE 24.69229
it: 200, train mse: 22.564250946044922, local kl: 0.0 global kl: 0.017527341842651367 valid mse: 20.14684295654297, local kl: 0.0 global kl: 0.09147223830223083
Saving best model with MSE 20.146843
it: 250, train ms[...]<jupyter_text>ANP<jupyter_code>model_type = 'anp'
x_y_encoder_net_sizes = [HIDDEN_SIZE]*2
global_latent_net_sizes = [HIDDEN_SIZE]*2
local_latent_net_sizes = None

model_hparams = tf.contrib.training.HParams(activation=tf.nn.relu,
                                            output_activation=tf.nn.relu,
                                            x_encoder_net_sizes=x_encoder_net_sizes,
                                            x_y_encoder_net_sizes=x_y_encoder_net_sizes,
                                            global_latent_net_sizes=global_latent_net_sizes,
                                            local_latent_net_sizes=local_latent_net_sizes,
                                            decoder_net_sizes=decoder_net_sizes, 
                                            heteroskedastic_net_sizes=heteroskedastic_net_sizes,
                                            att_type=att_type,
                                            att_heads=att_heads,
                                            model_type=model_type,
                                            data_uncertainty=data_uncertainty)
save_path = os.path.join(savedir, 'gnp_' + model_type + '.ckpt')
training_hparams = tf.contrib.training.HParams(lr=0.01,
                                               optimizer=tf.train.RMSPropOptimizer,
                                               num_iterations=10000,
                                               batch_size=10,
                                               num_context=num_context,
                                               num_target=num_target, 
                                               print_every=50,
                                               save_path=save_path,
                                               max_grad_norm=1000.0)

train(data_hparams,
      model_hparams,
      training_hparams)<jupyter_output>it: 0, train mse: 64.7630081177, local kl: 0.0 global kl: 0.0284446384758 valid mse: 79.3176803589, local kl: 0.0 global kl: 0.0269404239953
Saving best model with MSE 79.31768
it: 50, train mse: 49.9610214233, local kl: 0.0 global kl: 0.0187800955027 valid mse: 68.77003479, local kl: 0.0 global kl: 0.00885808933526
Saving best model with MSE 68.770035
it: 100, train mse: 27.6876678467, local kl: 0.0 global kl: 0.00339605892077 valid mse: 25.6078796387, local kl: 0.0 global kl: 0.00314322533086
Saving best model with MSE 25.60788
it: 150, train mse: 16.9543037415, local kl: 0.0 global kl: 0.00767825264484 valid mse: 29.4613399506, local kl: 0.0 global kl: 0.00151743693277
it: 200, train mse: 37.5948066711, local kl: 0.0 global kl: 0.00202285428531 valid mse: 20.3689975739, local kl: 0.0 global kl: 0.00270827626809
Saving best model with MSE 20.368998
it: 250, train mse: 22.1894664764, local kl: 0.0 global kl: 0.00616811122745 valid mse: 30.6764945984, local kl: 0.0 global kl: 0.0049195[...]<jupyter_text>ACNP<jupyter_code>model_type = 'acnp'
x_y_encoder_net_sizes = [HIDDEN_SIZE]*4
global_latent_net_sizes = None
local_latent_net_sizes = None

model_hparams = tf.contrib.training.HParams(activation=tf.nn.relu,
                                            output_activation=tf.nn.relu,
                                            x_encoder_net_sizes=x_encoder_net_sizes,
                                            x_y_encoder_net_sizes=x_y_encoder_net_sizes,
                                            global_latent_net_sizes=global_latent_net_sizes,
                                            local_latent_net_sizes=local_latent_net_sizes,
                                            decoder_net_sizes=decoder_net_sizes, 
                                            heteroskedastic_net_sizes=heteroskedastic_net_sizes,
                                            att_type=att_type,
                                            att_heads=att_heads,
                                            model_type=model_type,
                                            data_uncertainty=data_uncertainty)
save_path = os.path.join(savedir, 'gnp_' + model_type + '.ckpt')
training_hparams = tf.contrib.training.HParams(lr=0.01,
                                               optimizer=tf.train.RMSPropOptimizer,
                                               num_iterations=10000,
                                               batch_size=10,
                                               num_context=num_context,
                                               num_target=num_target, 
                                               print_every=50,
                                               save_path=save_path,
                                               max_grad_norm=1000.0)

train(data_hparams,
      model_hparams,
      training_hparams)<jupyter_output>it: 0, train mse: 65.7367477417, local kl: 0.0 global kl: 0.0 valid mse: 81.7034683228, local kl: 0.0 global kl: 0.0
Saving best model with MSE 81.70347
it: 50, train mse: 65.3536453247, local kl: 0.0 global kl: 0.0 valid mse: 78.2391357422, local kl: 0.0 global kl: 0.0
Saving best model with MSE 78.239136
it: 100, train mse: 20.2306365967, local kl: 0.0 global kl: 0.0 valid mse: 24.6245040894, local kl: 0.0 global kl: 0.0
Saving best model with MSE 24.624504
it: 150, train mse: 25.3244667053, local kl: 0.0 global kl: 0.0 valid mse: 29.9224376678, local kl: 0.0 global kl: 0.0
it: 200, train mse: 55.0799636841, local kl: 0.0 global kl: 0.0 valid mse: 37.4522705078, local kl: 0.0 global kl: 0.0
it: 250, train mse: 19.7182826996, local kl: 0.0 global kl: 0.0 valid mse: 20.1678028107, local kl: 0.0 global kl: 0.0
Saving best model with MSE 20.167803
it: 300, train mse: 17.832529068, local kl: 0.0 global kl: 0.0 valid mse: 28.3981285095, local kl: 0.0 global kl: 0.0
it: 350, train mse: 18.4[...]<jupyter_text>ACNS<jupyter_code>model_type = 'acns'
x_y_encoder_net_sizes = [HIDDEN_SIZE]*2
global_latent_net_sizes = [HIDDEN_SIZE]*2
local_latent_net_sizes = [HIDDEN_SIZE]*2


model_hparams = tf.contrib.training.HParams(activation=tf.nn.relu,
                                            output_activation=tf.nn.relu,
                                            x_encoder_net_sizes=x_encoder_net_sizes,
                                            x_y_encoder_net_sizes=x_y_encoder_net_sizes,
                                            global_latent_net_sizes=global_latent_net_sizes,
                                            local_latent_net_sizes=local_latent_net_sizes,
                                            decoder_net_sizes=decoder_net_sizes, 
                                            heteroskedastic_net_sizes=heteroskedastic_net_sizes,
                                            att_type=att_type,
                                            att_heads=att_heads,
                                            model_type=model_type,
                                            data_uncertainty=data_uncertainty)
save_path = os.path.join(savedir, 'gnp_' + model_type + '.ckpt')
training_hparams = tf.contrib.training.HParams(lr=0.01,
                                               optimizer=tf.train.RMSPropOptimizer,
                                               num_iterations=10000,
                                               batch_size=10,
                                               num_context=num_context,
                                               num_target=num_target, 
                                               print_every=50,
                                               save_path=save_path,
                                               max_grad_norm=1000.0)

train(data_hparams,
      model_hparams,
      training_hparams)<jupyter_output>it: 0, train mse: 62.2864456177, local kl: 5499.07910156 global kl: 0.0314277894795 valid mse: 79.5546112061, local kl: 9059.21679688 global kl: 0.0417891927063
Saving best model with MSE 79.55461
it: 50, train mse: 59.0019950867, local kl: 0.181515619159 global kl: 0.125239357352 valid mse: 62.9373626709, local kl: 0.0683052390814 global kl: 0.284775853157
Saving best model with MSE 62.937363
it: 100, train mse: 35.8874435425, local kl: 1.09855043888 global kl: 0.0940564647317 valid mse: 45.1217079163, local kl: 0.506570696831 global kl: 0.269403219223
Saving best model with MSE 45.121708
it: 150, train mse: 19.0480823517, local kl: 0.255944609642 global kl: 0.00604878785089 valid mse: 17.5246601105, local kl: 0.260885983706 global kl: 0.0064511760138
Saving best model with MSE 17.52466
it: 200, train mse: 42.1990890503, local kl: 0.22990898788 global kl: 0.00508384406567 valid mse: 41.239238739, local kl: 0.213793352246 global kl: 0.00565440673381
it: 250, train mse: 17.5339736938, l[...]<jupyter_text>Freeform<jupyter_code>model_type = 'fully_connected'
x_y_encoder_net_sizes = [HIDDEN_SIZE]*2
global_latent_net_sizes = [HIDDEN_SIZE]*2
local_latent_net_sizes = [HIDDEN_SIZE]*2

model_hparams = tf.contrib.training.HParams(activation=tf.nn.relu,
                                            output_activation=tf.nn.relu,
                                            x_encoder_net_sizes=x_encoder_net_sizes,
                                            x_y_encoder_net_sizes=x_y_encoder_net_sizes,
                                            global_latent_net_sizes=global_latent_net_sizes,
                                            local_latent_net_sizes=local_latent_net_sizes,
                                            decoder_net_sizes=decoder_net_sizes, 
                                            heteroskedastic_net_sizes=heteroskedastic_net_sizes,
                                            att_type=att_type,
                                            att_heads=att_heads,
                                            model_type=model_type,
                                            data_uncertainty=data_uncertainty)
save_path = os.path.join(savedir, 'gnp_' + model_type + '.ckpt')
training_hparams = tf.contrib.training.HParams(lr=0.01,
                                               optimizer=tf.train.RMSPropOptimizer,
                                               num_iterations=10000,
                                               batch_size=10,
                                               num_context=num_context,
                                               num_target=num_target, 
                                               print_every=50,
                                               save_path=save_path,
                                               max_grad_norm=1000.0)

train(data_hparams,
      model_hparams,
      training_hparams)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.06427230046948357, 'fraction_numerical': 0.09342723004694836, 'mean_word_length': 1.3906846240179573, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 318, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '224422', 'license': 'Apache-2.0', 'n_tokens_mistral': 7098, 'n_tokens_neox': 5753, 'n_words': 1107, 'path': 'experimental/attentive_uncertainty/colabs/2019_09_10_gnp_pretrain.ipynb', 'repo_name': 'debbiemarkslab/edward2', 'size': '21300'}