<jupyter_start><jupyter_text>kernel_rate = 0 means leave kernel to values I set turn natgrad_rate down to 0.01 if elbo looks wonky if doing 1e6 data points make batch size bigger step_decay, after every epoch natgrad_rate multiplied by<jupyter_code>xmesh_low, ymesh_low, zmesh_low, xgrid_low, fgrid_low = genDustDensity(nx_induce, nx_induce, nz_induce, scale, scale, zscale)
xmesh_high, ymesh_high, zmesh_high, xgrid_high, fgrid_high = genDustDensity(nx, ny, nz, scale, scale, zscale)
xmesh_low.shape
fig, axes = plt.subplots(1, 2, figsize=(12,5))

ax = axes[0]
n = 2
im = ax.pcolormesh(xmesh_low[:,:,n], ymesh_low[:,:,n], fgrid_low[:,:,n].T)
plt.colorbar(im, ax=ax)

ax = axes[1]
im = ax.pcolormesh(xmesh_high[:,:,n*4], ymesh_high[:,:,n*4], fgrid_high[:,:,n*4].T)
plt.colorbar(im, ax=ax)

fig, axes = plt.subplots(1, 2, figsize=(12,5))

ax = axes[0]
n = 10
im = ax.pcolormesh(xmesh[:,:,n], ymesh[:,:,n], fgrid[:,:,n].T)
plt.colorbar(im, ax=ax)

z = np.unique(zmesh[:,:,n])

ax = axes[1]
inslice = np.abs(xtest[:,2] - z) < 0.01
im = ax.scatter(xtest[inslice,0], xtest[inslice,1], c=ftest[inslice], s=10)
plt.colorbar(im, ax=ax)
output_dir = "./monte-carlo-gradient-domainsimulation-output_1kpc"
fit_kwargs = {'epochs'        : 2,
              'natgrad_rate'  : 1e-1,
              'kernel_rate'   : 0, #1e-6,
              'integrated_obs': True,
              'batch_callback': None, 
              'batch_size'    : 5000,
              'step_decay'    : .9,
              'sig2_init'     : 2,
              'ell_init'      : 0.07}

#########################################################################
# Example: Integrated GP model using Squared Exp Kernel (analytical)    #
#########################################################################
eu.fit_model_and_save("SqExp-analytic",
      xobs=xobs, aobs=aobs[:,None], sobs=sobs[:,None],
      kern=svgp.SqExp(), xinduce=xinduce,
      xtest=xtest, etest=etest, ftest=ftest*50.,
      xgrid=xgrid, fgrid=fgrid*50.,
      output_dir=output_dir,
      **fit_kwargs)

model_name = 'monte-carlo-gradient-domainsimulation-output_1kpc/SqExp-analytic'
n=10
z = np.unique(zmesh[:,:,n])
inslice = np.abs(xobs[:,2] - z) < 0.01
plot_grid(model_name, fgrid[:,:,n].value*50., xobs[inslice], aobs[inslice], n, xx1=xmesh, xx2=ymesh,pretty_name='SpExp-analytic')
#eu.plot_posterior_grid(model_name, xx1=xmesh, xx2=ymesh, pretty_name='SpExp-analytic')
eu.make_zscore_histogram(model_name, pretty_name='SpExp-analytic')
eu.make_model_distance_plots(model_name, pretty_name='SpExp-analytic')
eu.make_error_plots(output_dir, model_names=[model_name], pretty_names=['SpExp-analytic'])
eu.make_qq_plots([model_name], pretty_names=['SpExp-analytic'])
predictions = torch.load('monte-carlo-gradient-domainsimulation-output/SqExp-analytic/predictions.pkl')
np.max(predictions['fmu_test'])
np.sum(predictions['fmu_test'] == 0)
predictions['fmu_test'][predictions['fmu_test'] != 0]
plt.plot(predictions['fmu_test'])<jupyter_output><empty_output><jupyter_text>Dataframe objects for a bunch of different kernels, including Sara's with her hyper parameters At least 1M observations<jupyter_code>f['data']['px_true']
def print_attrs(name, obj):
    print(name)
    for key, val in obj.attrs.iteritems():
        print("    %s: %s" % (key, val))

f.visititems(print_attrs)<jupyter_output>data
{'fraction_non_alphanumeric': 0.2000601684717208, 'fraction_numerical': 0.021359807460890494, 'mean_word_length': 5.157407407407407, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '577196', 'license': 'MIT', 'n_tokens_mistral': 1368, 'n_tokens_neox': 1294, 'n_words': 222, 'path': 'SVIGPdomainsim.ipynb', 'repo_name': 'andersdot/dust', 'size': '3324'}