<jupyter_start><jupyter_text>This notebook will make plots of the emu accuracy and compare it to the other error estimates. Considering how I want to make these. I am currently leaning toward showing fractional accuracy against the test sample. Could also show the covmats/etc but not worried about that RN. I also am considering showing the emu prediction of the test sample I had shown, or even a fit to it? Dunno if thats necessary.<jupyter_code>from matplotlib import pyplot as plt
%matplotlib inline
import seaborn as sns
#sns.set()
#import matplotlib.colors as colors
sns.set_palette('deep')
plt.rcParams.update({'font.size': 18})
import numpy as np
from pearce.emulator import LemonPepperWet
from glob import  glob
from os import path
current_palette = sns.color_palette()
model_color_map = {'HOD': (current_palette[0], "GnBu_d"),
                    'HSAB': (current_palette[1], "YlGn_d"),
                   'SHAM': (current_palette[2], "OrRd_d"),
                   'CAB': (current_palette[3], "RdPu_d"),
                   'CorrAB': (current_palette[4], "YlOrBr_d"),
                    'Halos': (current_palette[5], 'PuBu_d')}
models = ['CAB', 'CorrAB', 'HSAB', 'HOD']
all_fnames = sorted(glob('/u/ki/swmclau2/des/Aemulators/%s_*_h/*.hdf5'%'ds'))
print all_fnames
accs = np.zeros((4,2,18))
errs = np.zeros((4,2,18))

for j,obs in enumerate(['wp', 'ds']):
    all_fnames = sorted(glob('/u/ki/swmclau2/des/Aemulators/%s_*_h/*.hdf5'%obs))
    test_fnames, train_fnames = [],[]

    for f in all_fnames:
        if 'conc' in f:
            continue
        if 'test' in f:
            test_fnames.append(f)
        else:
            train_fnames.append(f)
    

    for i, (train_fname, test_fname) in enumerate(zip(train_fnames, test_fnames)):
        print train_fname, test_fname
        #print
        try:
            emu = LemonPepperWet(train_fname, fixed_params = {'z':0.0})
        except:
            continue
        
        #print train_fname, test_fname
        pred_y, data_y = emu.goodness_of_fit(test_fname, statistic=None)
        
        #print data_y, pred_y
        accs[i,j] = (np.abs(10**pred_y-10**data_y)/np.abs(10**data_y)).mean(axis=1)
        errs[i,j] = np.mean((10**pred_y-10**data_y)**2, axis=1)
        #print '*'*30
        if i>0:
            break
    break
R = (10**pred_y - 10**data_y)/(10**data_y)<jupyter_output><empty_output><jupyter_text>for i in xrange(18):    plt.hist(R[i], bins = 100)    plt.show() param_names = emu.get_param_names()param_bounds = [emu.get_param_bounds(p) for p in param_names]random_point = [np.random.uniform(l,h) for l,h in param_bounds]point = dict(zip(param_names, random_point)) %%timeit emu.emulate_wrt_r(point) print accs.mean(axis=2) errs.shape train_fnames<jupyter_code>full_cov = np.load('/u/ki/swmclau2/Git/pearce/bin/covmat/wp_ds_full_covmat_h.npy')#, full_cov[:36][:, :36])
#np.save('/home/users/swmclau2/Git/pearce/bin/covmat/wp_full_covmat.npy', full_cov[:18][:, :18])
#np.save('/home/users/swmclau2/Git/pearce/bin/covmat/ds_full_covmat.npy', full_cov[18:36][:, 18:36])
cov = np.load('/u/ki/swmclau2/Git/pearce/bin/covmat/wp_ds_sim_covmat_h.npy')#, cov[:36][:, :36])
error = np.sqrt(np.diag(cov)).reshape((-1,2), order ='F')
shape_error =  np.sqrt(np.diag(full_cov-cov)).reshape((-1,2), order ='F')
plt.plot(emu.scale_bin_centers, R.mean(axis=1))
plt.plot(emu.scale_bin_centers, error[:,0]/(10**data_y).mean(axis=1))
plt.xscale('log')
#emu covs
emu_cov_fnames = ['/u/ki/swmclau2/Git/pearce/bin/optimization/wp_hod_emu_cov_lpw.npy',
                  '/u/ki/swmclau2/Git/pearce/bin/optimization/ds_hod_emu_cov_lpw.npy']
emu_cov = np.zeros_like(full_cov[:36][:, :36])

for i, fname in enumerate(emu_cov_fnames):
    
    emu_cov[i*18:(i+1)*18][:, i*18:(i+1)*18] = np.load(fname)
def cov_to_corr(cov):
    std = np.sqrt(np.diag(cov))
    
    denom = np.outer(std, std)
    
    return cov/denom
cmap = sns.diverging_palette(240, 10, n=7, as_cmap = True)
plt.imshow(cov_to_corr(cov), cmap = cmap, vmin = -1)
plt.imshow(cov_to_corr(full_cov), cmap = cmap, vmin = -1)
plt.imshow(cov_to_corr(emu_cov), cmap = cmap, vmin = -1)
emu_err = np.sqrt(np.diag(emu_cov)).reshape((-1,2), order = 'F')
avg_ys = []
for j,obs in enumerate(['wp', 'ds']):
    train_fname = glob('/u/ki/swmclau2/des/Aemulators/%s_zheng07_h/*.hdf5'%obs)[0]
    emu = LemonPepperWet(train_fname, fixed_params = {'z':0.0})
    avg_ys.append(10**(emu.y.mean(axis=(0,1))+emu._y_mean) )<jupyter_output><empty_output><jupyter_text>import h5pyfor j,obs in enumerate(['wp', 'ds']):    train_fname = glob('/u/ki/swmclau2/des/Aemulators/%s_zheng07_h/*.hdf5'%obs)[0]    f = h5py.File(train_fname, 'r')    all_covs= f['cosmo_no_00']['a_1.000']['cov'][()]    break for i in xrange(2):    plt.plot(emu.scale_bin_centers, error[:, i]/(avg_ys[i]), label = 'Sim')    plt.plot(emu.scale_bin_centers,emu_err[:, i]/(avg_ys[i]), label = 'Emu')    plt.legend(loc='best')    plt.xscale('log');    plt.loglog();    plt.ylim([0.01, 0.2])    plt.show();<jupyter_code>models2 = sorted(['hod', 'hsab', 'cab', 'corrab'])<jupyter_output><empty_output><jupyter_text>fig, axes = plt.subplots(2,1, figsize = (7,2*4), sharex=True)emu_cov_fname = '/u/ki/swmclau2/Git/pearce/bin/optimization/%s_%s_emu_cov_lpw.npy'for i, (model, model_lc) in enumerate(zip(models, models2)):    print model, model_lc    for j, obs in enumerate(['wp', 'ds']):        emu_err = np.sqrt(np.diag(np.load(emu_cov_fname%(obs, model_lc))))        axes[j].plot(emu.scale_bin_centers, np.sqrt(errs[i,j])/(10**avg_ys[j]), color = model_color_map[model][0], label = model)        axes[j].plot(emu.scale_bin_centers, emu_err,                      color = model_color_map[model][0], label = model)        axes[j].set_ylim([5e-3*100, 4e-1*100])        axes[j].loglog();        axes[j].set_ylabel('%s Error'%obs)axes[1].legend(loc='best', frameon=False)axes[0].text(.5,.9, r'$w_p(r_p)$',                horizontalalignment='center',                transform=axes[0].transAxes)axes[1].text(.5,.9, r'$\Delta \Sigma(r_p)$',                horizontalalignment='center',                transform=axes[1].transAxes)for i,obs in enumerate(['wp', 'ds']):    axes[i].plot(emu.scale_bin_centers, np.sqrt(error[:,i]),/(10**avg_ys[i]),                  color = 'k', ls ='--')    axes[1].set_xlabel(r'$r_p$ [Mpc/h]')fig.tight_layout()<jupyter_code>import matplotlib.ticker as mticker

fig, axes = plt.subplots(2,1, figsize = (7,2*4), sharex=True)
emu_cov_fname = '/u/ki/swmclau2/Git/pearce/bin/optimization/%s_%s_emu_cov_lpw.npy'
for i, (model, model_lc) in enumerate(zip(models, models2)):
    for j, obs in enumerate(['wp', 'ds']):
        emu_err = np.sqrt(np.diag(np.load(emu_cov_fname%(obs, model_lc))))

        #axes[j].plot(emu.scale_bin_centers, np.sqrt(errs[i,j]), color = model_color_map[model][0], label = model)
        if j==0:
            print model, obs, emu_err/(avg_ys[j])
            print 
            
        axes[j].plot(emu.scale_bin_centers, emu_err/(avg_ys[j]), 
                     color = model_color_map[model][0], label = model)
        axes[j].set_ylim([0.005, 0.25])
        axes[j].loglog();
        axes[j].set_ylabel('Rel. Error')
        axes[j].yaxis.set_major_formatter(mticker.ScalarFormatter())

print '*'*20
print 'Sample', 'wp', error[:,0]/(avg_ys[0])

axes[0].text(.5,.9, r'$w_p(r_p)$',
                horizontalalignment='center',
                transform=axes[0].transAxes)
axes[1].text(.5,.9, r'$\Delta \Sigma(r_p)$',
                horizontalalignment='center',
                transform=axes[1].transAxes)

for i,obs in enumerate(['wp', 'ds']):
    axes[i].plot(emu.scale_bin_centers, error[:,i]/(avg_ys[i]),
                  color = 'k', ls ='--', label = 'Sample Var.')
    
axes[1].plot(emu.scale_bin_centers, np.sqrt(shape_error[:,1])/avg_ys[1],
             color ='k', ls =':', label ='Shape Noise')
#axes[0].legend(loc='best', frameon=False,fontsize='x-small')

axes[1].legend(loc=2, frameon=False,fontsize='x-small')

axes[1].set_xlabel(r'$r_p$ [Mpc/h]')
axes[1].xaxis.set_major_formatter(mticker.ScalarFormatter())
fig.tight_layout()
plt.savefig('emu_acc.pdf', format='pdf')<jupyter_output>CAB wp [0.03309503 0.02720104 0.02316457 0.02046791 0.02000029 0.02003478
 0.01988617 0.02039421 0.02138969 0.02185198 0.02180618 0.02484388
 0.02913308 0.0288154  0.02777096 0.03348236 0.04234706 0.07079702]

CorrAB wp [0.04903321 0.03579447 0.02447448 0.01661117 0.01673871 0.01983759
 0.02070867 0.01952998 0.01841381 0.01753921 0.01576653 0.01660646
 0.02180537 0.02360919 0.02369265 0.02996687 0.04234369 0.08232241]

HSAB wp [0.02307005 0.0202891  0.01854829 0.0168724  0.01586562 0.01458477
 0.01292761 0.01202739 0.01359317 0.01380736 0.01253044 0.01374444
 0.01677331 0.01879177 0.02309737 0.03063953 0.0421578  0.07334102]

HOD wp [0.0648127  0.04884343 0.02897523 0.01915321 0.02207054 0.02790297
 0.02718875 0.02325624 0.01930038 0.01585668 0.01215685 0.01243648
 0.01503882 0.01533264 0.01875476 0.02667975 0.03918428 0.07256145]

********************
Sample wp [0.01486263 0.01426745 0.01421419 0.01446001 0.01466129 0.01517814
 0.01567564 0.01628523 0.01622986 0.01553344 0.01450395 0.[...]
{'fraction_non_alphanumeric': 0.17641902257286873, 'fraction_numerical': 0.10453648915187377, 'mean_word_length': 3.803684210526316, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '470942', 'license': 'MIT', 'n_tokens_mistral': 4381, 'n_tokens_neox': 3733, 'n_words': 612, 'path': 'notebooks/Aemulus V Emu Accuracy Plots.ipynb', 'repo_name': 'mclaughlin6464/pearce', 'size': '9126'}