<jupyter_start><jupyter_text>Load data<jupyter_code>import os
import numpy as np
from lib.eval.regression import normalize
from lib.zero_shot import get_gap_ids
from lib.utils import mkdir_p

# setup
seed = 123
rng = np.random.RandomState(seed)
data_dir = 'data/' #'../wgan/data/' 
codes_dir = os.path.join(data_dir, 'codes/')
figs_dir = 'figs/'
mkdir_p(figs_dir)

n_c = 10
zshot = True

model_names = ['PCA', 'VAE', '$\\beta$-VAE', 'InfoGAN']
exp_names = [m.lower() for m in model_names]
n_models = len(model_names)
train_fract, dev_fract, test_fract = 0.8, 0.1, 0.1

# load inputs (model codes)
m_codes = []
for n in exp_names:
    try:
        m_codes.append(np.load(os.path.join(codes_dir, n + '.npy')))
    except IOError:
        # .npz, e.g. pca with keys: codes, explained_variance
        m_codes.append(np.load(os.path.join(codes_dir, n + '.npz'))['codes'])

# load targets (ground truths)
gts = np.load(os.path.join(data_dir, 'teapots.npz'))['gts']
n_samples = gts.shape[0]
n_z = gts.shape[1]
n_train, n_dev, n_test = int(train_fract*n_samples), int(dev_fract*n_samples), int(test_fract*n_samples)

# create 'gap' in data if zeroshot (unseen factor combinations)
if zshot:
    try:
        gap_ids = np.load(os.path.join(data_dir, 'gap_ids.npy'))
    except IOError:
        gap_ids = get_gap_ids(gts)
    def create_gap(data):
        return np.delete(data, gap_ids, 0)
    
# split inputs and targets into sets: [train, dev, test, (zeroshot)]
def split_data(data):
    train = data[:n_train]
    dev = data[n_train: n_train + n_dev]
    test = data[n_train + n_dev: n_train + n_dev + n_test]
    if zshot:
        return [create_gap(train), create_gap(dev), create_gap(test), data[gap_ids]]
    return [train, dev, test, None]

gts = split_data(gts)
for i in range(n_models):
    m_codes[i] = split_data(m_codes[i])    

# normalize input and target datasets [train, dev, test, (zeroshot)]
def normalize_datasets(datasets):
    datasets[0], mean, std, _ = normalize(datasets[0], remove_constant=False)
    datasets[1], _, _, _ = normalize(datasets[1], mean, std, remove_constant=False)
    datasets[2], _, _, _ = normalize(datasets[2], mean, std, remove_constant=False)
    if zshot:
        datasets[3], _, _, _ = normalize(datasets[3], mean, std, remove_constant=False)
    return datasets

gts = normalize_datasets(gts)
for i in range(n_models):
    m_codes[i] = normalize_datasets(m_codes[i])<jupyter_output>/home/cian/miniconda3/envs/style/lib/python3.6/site-packages/ipykernel_launcher.py:46: DeprecationWarning: in the future out of bounds indices will raise an error instead of being ignored by `numpy.delete`.<jupyter_text>Regression  Fit regressor, visualise and quantify criteria<jupyter_code>import matplotlib.pyplot as plt
# %matplotlib inline
from lib.eval.hinton import hinton
from lib.eval.regression import *

def fit_visualise_quantify(regressor, params, err_fn, importances_attr, test_time=False, save_plot=False):
    # lists to store scores
    m_disent_scores   = [] * n_models
    m_complete_scores = [] * n_models
    
    # arrays to store errors (+1 for avg)
    train_errs = np.zeros((n_models, n_z + 1))
    dev_errs   = np.zeros((n_models, n_z + 1))
    test_errs  = np.zeros((n_models, n_z + 1)) 
    zshot_errs = np.zeros((n_models, n_z + 1))
    
    # init plot (Hinton diag)
    fig, axs = plt.subplots(1,n_models, figsize=(12, 6), facecolor='w', edgecolor='k')
    axs = axs.ravel()
    
    for i in range(n_models):
        # init inputs
        X_train, X_dev, X_test, X_zshot = m_codes[i][0], m_codes[i][1], m_codes[i][2], m_codes[i][3]
           
        # R_ij = relative importance of c_i in predicting z_j
        R = [] 
        
        for j in range(n_z):
            # init targets [shape=(n_samples, 1)]
            y_train = gts[0][:, j]
            y_dev   = gts[1][:, j]
            y_test  = gts[2][:, j] if test_time else None
            y_zshot = gts[3][:, j] if zshot else None
            
            # fit model
            model = regressor(**params[i][j])
            model.fit(X_train, y_train)

            # predict
            y_train_pred = model.predict(X_train)
            y_dev_pred   = model.predict(X_dev)
            y_test_pred  = model.predict(X_test) if test_time else None
            y_zshot_pred = model.predict(X_zshot) if zshot else None
            
            # calculate errors
            train_errs[i, j] = err_fn(y_train_pred, y_train)
            dev_errs[i, j]   = err_fn(y_dev_pred, y_dev)
            test_errs[i, j]  = err_fn(y_test_pred, y_test) if test_time else None
            zshot_errs[i, j] = err_fn(y_zshot_pred, y_zshot) if zshot else None            
            
            # extract relative importance of each code variable in predicting z_j
            r = getattr(model, importances_attr)[:, None] # [n_c, 1]
            R.append(np.abs(r))

        R = np.hstack(R) #columnwise, predictions of each z

        # disentanglement
        disent_scores = entropic_scores(R.T)
        c_rel_importance = np.sum(R,1) / np.sum(R) # relative importance of each code variable
        disent_w_avg = np.sum(np.array(disent_scores) * c_rel_importance)
        disent_scores.append(disent_w_avg)
        m_disent_scores.append(disent_scores)

        # completeness
        complete_scores = entropic_scores(R)
        complete_avg = np.mean(complete_scores)
        complete_scores.append(complete_avg)
        m_complete_scores.append(complete_scores)

        # informativeness (append averages)
        train_errs[i, -1] = np.mean(train_errs[i, :-1])
        dev_errs[i, -1]   = np.mean(dev_errs[i, :-1])
        test_errs[i, -1]  = np.mean(test_errs[i, :-1]) if test_time else None
        zshot_errs[i, -1] = np.mean(zshot_errs[i, :-1]) if zshot else None

        # visualise
        hinton(R, '$\mathbf{z}$', '$\mathbf{c}$', ax=axs[i], fontsize=18)
        axs[i].set_title('{0}'.format(model_names[i]), fontsize=20)
    
    plt.rc('text', usetex=True)
    if save_plot:
        fig.tight_layout()
        plt.savefig(os.path.join(figs_dir, "hint_{0}_{1}.pdf".format(regressor.__name__, n_c)))
    else:
        plt.show()

    print_table_pretty('Disentanglement', m_disent_scores, 'c', model_names)
    print_table_pretty('Completeness', m_complete_scores, 'z', model_names)

    print("Informativeness:")
    
    print_table_pretty('Training Error', train_errs, 'z', model_names)
    print_table_pretty('Validation Error', dev_errs, 'z', model_names)
    
    if test_time:
        print_table_pretty('Test Error', test_errs, 'z', model_names)
        if zshot:
            print_table_pretty('Zeroshot Error', zshot_errs, 'z', model_names)<jupyter_output><empty_output><jupyter_text>Lasso<jupyter_code>from sklearn.linear_model import Lasso

alpha = 0.02
params = [[{"alpha": alpha}] * n_z] * n_models # constant alpha for all models and targets
importances_attr = 'coef_' # weights
err_fn = nrmse # norm root mean sq. error
test_time = True
save_plot = False

fit_visualise_quantify(Lasso, params, err_fn, importances_attr, test_time, save_plot)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>from sklearn.ensemble.forest import RandomForestRegressor

n_estimators = 10
all_best_depths = [[12, 10, 10, 10, 10] , [12, 10, 3, 3, 3], [12, 10, 3, 3, 3], [4, 5, 2, 5, 5]]

# populate params dict with best_depths per model per target (z gt)
params = [[]] * n_models
for i, z_max_depths in enumerate(all_best_depths):
    for z_max_depth in z_max_depths:
        params[i].append({"n_estimators":n_estimators, "max_depth":z_max_depth, "random_state": rng})

importances_attr = 'feature_importances_'
err_fn = nrmse # norm root mean sq. error
test_time = True
save_plot = False

fit_visualise_quantify(RandomForestRegressor, params, err_fn, importances_attr, test_time, save_plot)<jupyter_output><empty_output><jupyter_text>Figs  z vs. c<jupyter_code>from matplotlib.transforms import offset_copy

zs = [0,0] + list(range(n_z))
all_import_codes = [[5,8,4,0,2,1,1],[2,5,9,6,1,8,3],[5,7,2,6,1,9,3],[0,8,1,3,4,9,2]]
n_samples = 5000
fig, axs = plt.subplots(len(zs), n_models, figsize=(20, 25), facecolor='w', edgecolor='k', sharey=True, sharex=True)

for i, import_codes in zip(range(n_models), all_import_codes):
    X_train = m_codes[i][0]    
    for j, (z, c) in enumerate(zip(zs, import_codes)):
        X = X_train[:, c:c+1]
        y = gts[0][:, z]
        X, y = subset_of_data(X, y, n_samples)
        
        if i == 0: # set column titles
            axs[j,i].set_ylabel('$z_{0}$'.format(z), fontsize=28)
        
        if j == 0:
            axs[j,i].set_title('{0}'.format(model_names[i]), fontsize=28)

        axs[j,i].set_xlabel('$c_{0}$'.format(c), fontsize=28)
        axs[j,i].scatter(y, X, color='black', linewidth=0)        
        axs[j,i].legend(loc=1, fontsize=21)
        axs[j,i].set_ylim([-3.5,3.5])
        axs[j,i].set_xlim([-2,2])
        axs[j,i].grid(True)
        axs[j,i].set_axisbelow(True)

plt.rc('text', usetex=True)
fig.tight_layout()
#plt.show()
plt.savefig(os.path.join(figs_dir, "cvsz.pdf"))<jupyter_output><empty_output><jupyter_text>Visual disentanglement<jupyter_code>import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import os
#%matplotlib inline

base = "/home/cian/wgan/papers/images"
models = ["vae", "bvae", "infogan"]
fs = ["traverse{0}".format(m) for m in models]
paths = [os.path.join(base, f) for f in fs]

zs = [0,0] + list(range(5))
all_important_cs = [[2,5,9,6,1,8,3],[5,7,2,6,1,9,3],[0,8,1,3,4,9,2]]
all_xlabels = [[-3, 0, 3], [-3, 0, 3], [-1, 0, 1]]
xlocs = [0, 320, 640]

ylabels = ['$z_{0}$'.format(z) for z in zs]
ylocs = [64*(z+1) - 32 for z in range(len(labels))]

for p, important_cs, xlabels in zip(paths, all_important_cs, all_xlabels):
    ylbls = [ylabels[i] + "$(c_{0})$".format(important_cs[i]) for i in range(len(ylabels))]
    image = mpimg.imread(p + ".png")
    plt.imshow(image)
    plt.yticks(ylocs, ylbls)
    plt.xticks(xlocs, xlabels, fontsize=8)
    plt.tight_layout()
    plt.savefig(os.path.join(base, p + ".pdf"), dpi=700)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14339473163002575, 'fraction_numerical': 0.020796197266785502, 'mean_word_length': 3.52666965486329, 'pattern_counts': {'":': 4, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '46603', 'license': 'MIT', 'n_tokens_mistral': 4058, 'n_tokens_neox': 3889, 'n_words': 804, 'path': 'quantify.ipynb', 'repo_name': 'cianeastwood/qedr', 'size': '10098'}