<jupyter_start><jupyter_text>OAE: MNIST Evaluations - Numerics<jupyter_code>import os
import sys
import logging
import json
import copy
import random
import numpy as np
import pandas as pd

os.environ['CUDA_VISIBLE_DEVICES']='0'
import tensorflow as tf

# seed_value = 202
seed_value = 1234
os.environ['PYTHONHASHSEED']=str(seed_value)
random.seed(seed_value)
np.random.seed(seed_value)
tf.set_random_seed(seed_value)

from keras.models import model_from_yaml, load_model, Model
from keras.layers import Input, Lambda
from keras.utils import to_categorical
import keras.backend as k

config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = True
k.set_session(tf.Session(config=config))

sys.path.append('/'.join(os.getcwd().split('/')))
from ornstein_auto_encoder import configuration
from ornstein_auto_encoder import readers
from ornstein_auto_encoder import samplers
from ornstein_auto_encoder import build_network
logging.basicConfig(format = '[%(name)-8s|%(levelname)s|%(filename)s:%(lineno)s] %(message)s',
                    level=logging.INFO)
log = logging.getLogger()
logging.getLogger("PIL").setLevel(logging.CRITICAL)
mnist_classifier = load_model('mnist_classifier.h5')
model_path = 'mnist_pretrained/mnist_imbalance_psoae'
models_aka = 'PSOAE'
path_info_config = "configurations/mnist/psoae_path_info.cfg"
network_info_config = "configurations/mnist/psoae_network_info.cfg"
feature_b = False

config_data = configuration.Configurator(path_info_config, log, verbose=False)
config_data.set_config_map(config_data.get_section_map())
config_network = configuration.Configurator(network_info_config, log, verbose=False)
config_network.set_config_map(config_network.get_section_map())
path_info = config_data.get_config_map()
network_info = config_network.get_config_map()

# Reader
reader_class = getattr(readers, network_info['model_info']['reader_class'].strip())  
reader = reader_class(log, path_info, network_info, verbose=False)

image_shape = reader.img_shape
img_pair1 = Input(shape=image_shape, name='image_input_1', dtype='float32')
img_pair2 = Input(shape=image_shape, name='image_input_2', dtype='float32')
ms_ssim = Lambda(lambda x: tf.image.ssim_multiscale(x[0], x[1], 1), name='ms_ssim')([img_pair1, img_pair2])
ms_ssim_model = Model([img_pair1, img_pair2], ms_ssim)

ssim = Lambda(lambda x: tf.image.ssim(x[0], x[1], 1), name='ssim')([img_pair1, img_pair2])
ssim_model = Model([img_pair1, img_pair2], ssim)

def get_ssim(class_images, size=50):
    pair1 = np.random.choice(np.arange(class_images.shape[0]), size)
    pair2 = np.random.choice(np.setdiff1d(np.arange(class_images.shape[0]), pair1, assume_unique=True), size)
    return ssim_model.predict_on_batch([class_images[pair1], class_images[pair2]])

def get_ms_ssim(class_images, size=50):
    pair1 = np.random.choice(np.arange(class_images.shape[0]), size)
    pair2 = np.random.choice(np.setdiff1d(np.arange(class_images.shape[0]), pair1, assume_unique=True), size)
    return ms_ssim_model.predict_on_batch([class_images[pair1], class_images[pair2]])
def get_numerics(model_path, model_aka, 
                 path_info_config = "configurations/mnist/psoae_path_info.cfg",
                 network_info_config = "configurations/mnist/psoae_network_info.cfg", feature_b=False):
    config_data = configuration.Configurator(path_info_config, log, verbose=False)
    config_data.set_config_map(config_data.get_section_map())
    config_network = configuration.Configurator(network_info_config, log, verbose=False)
    config_network.set_config_map(config_network.get_section_map())
    path_info = config_data.get_config_map()
    network_info = config_network.get_config_map()

    architecture = path_info['model_info']['model_architecture']

    ### Bulid network ####################################################################################
    network_class = getattr(build_network, network_info['model_info']['network_class'].strip())  
    network = network_class(log, path_info, network_info, n_label=reader.get_n_label())
    network.build_model('./%s/%s' % (model_path,  path_info['model_info']['model_architecture']), verbose=False)
    network.load(model_path)

    # Training
    train_tot_idxs_path = os.path.join(model_path, path_info['model_info']['train_tot_idxs'])
    test_tot_idxs_path = os.path.join(model_path, path_info['model_info']['test_tot_idxs'])
    train_idx = np.load(train_tot_idxs_path)
    test_idx = np.load(test_tot_idxs_path)

    ### Sampler ##########################################################################################
    log.info('-----------------------------------------------------------------')
    # Training data sampler
    log.info('Construct training data sampler')
    train_sampler_class = getattr(samplers, network_info['training_info']['sampler_class'].strip())
    train_sampler = train_sampler_class(log, train_idx, reader, network_info['training_info'], verbose=True)

    # Test data sampler
    log.info('Construct test data sampler')
    validation_sampler_class = getattr(samplers, network_info['validation_info']['sampler_class'].strip())
    test_sampler = validation_sampler_class(log, test_idx, reader, network_info['validation_info'], verbose=True)

    tot_original_accuracy = []
    tot_original_ssim_mean = []
    tot_sharpness_original = []

    tot_reconstruction = []
    tot_accuracy = []
    tot_gen_ssim_mean = []
    tot_sharpness_gen = []

    tot_one_shot_accuracy = []
    tot_one_shot_gen_ssim_mean = []
    tot_one_shot_sharpness_gen = []

    for nrepeat in range(10):
        log.info('------------------ %d repeat ------------------------------------------------------' % nrepeat)
        current_idxs = np.random.choice(test_idx, 1000)
        x, y = test_sampler.reader.get_batch(current_idxs)
        y_table = pd.Series(y)
        y_index = pd.Index(y)
        y_class = np.sort(y_table.unique())
        y_table = pd.Series(y)
        y_counts = y_table.value_counts()
        log.info('-------------------------------------------------')
        log.info('Images per Class')
        log.info('\n%s', y_counts)
        log.info('-------------------------------------------------')
        log.info('Summary')
        log.info('\n%s', y_counts.describe())
        log.info('-------------------------------------------------')
        wx, wy = network.main_sampler(x,y)
        try:
            if len(x.shape) == 4: real_img = x
        except: real_img = x[0]

        picked_one_shot_idxs_per_class = [np.random.choice(np.where(y == cls)[0], 1)[0] for i, cls in enumerate(y_class)]
        repeated = 100
        esp = 1.
        gen_y_class = np.repeat(y_class, repeated, axis=0)

        if ('randomintercept' in network_info['model_info']['network_class'].lower() or \
            'productspace' in network_info['model_info']['network_class'].lower()):
            recon_x = network.ae_model.predict(wx[:-1], batch_size=100, verbose=0)
        elif ('conditional' in network_info['model_info']['network_class'].lower()):
            recon_x = network.ae_model.predict(wx, batch_size=100, verbose=0)
        else:
            recon_x = network.ae_model.predict(wx[:1], batch_size=100, verbose=0)
        if 'randomintercept' in network_info['model_info']['network_class'].lower():
            b_sd = float(network_info['model_info']['b_sd'])
            estimate_b, fake_noise = network.encoder_model.predict(x, batch_size=100)
            new_b = np.random.multivariate_normal(np.zeros(network.get_z_dim()), 
                                                  b_sd**2.*np.identity(network.get_z_dim()), 
                                                  y_class.shape[0]).astype(np.float32)
            b = np.array([np.mean(estimate_b[np.random.choice(np.where(y==cls)[0],5)], axis=0) for cls in y_class])
            one_shot_b = np.array([estimate_b[picked_one_shot_idxs_per_class[i]] for i, cls in enumerate(y_class)])
            fake_latent = estimate_b + fake_noise
        elif 'productspace' in network_info['model_info']['network_class'].lower():
            if feature_b: img, feature, clss, b_noise = wx
            else: img, clss, b_noise = wx
            b_sd = float(network_info['model_info']['b_sd'])

            if feature_b:
                sample_b, b_given_x, estimate_b  = network.encoder_b_model.predict_on_batch([feature, clss])
            else:
                    sample_b, b_given_x, estimate_b  = network.encoder_b_model.predict_on_batch([img, clss])
            b = np.array([np.mean(estimate_b[np.random.choice(np.where(y==cls)[0],5)], axis=0) for cls in y_class])
            fake_noise = network.encoder_e_model.predict(wx[:-1], batch_size=100)
            new_b = np.random.multivariate_normal(np.zeros(b.shape[-1]), 
                                                  b_sd**2.*np.identity(b.shape[-1]), 
                                                  y_class.shape[0]).astype(np.float32)
            one_shot_b = np.array([estimate_b[picked_one_shot_idxs_per_class[i]] for i, cls in enumerate(y_class)])
        else:
            fake_latent = network.encoder_model.predict(x, batch_size=100)
            fake_noise = fake_latent

        mean = np.zeros(fake_noise.shape[-1])
        cov = float(network_info['model_info']['e_sd'])**2.*np.identity(fake_noise.shape[-1])
        true_noise = np.random.multivariate_normal(mean,cov,y.shape[0]).astype(np.float32)
        mean = np.zeros(fake_noise.shape[-1])
        cov = esp*np.identity(fake_noise.shape[-1])
        noise = np.random.multivariate_normal(mean,cov,y_class.shape[0]*repeated).astype(np.float32)

        if 'randomintercept' in network_info['model_info']['network_class'].lower():
            generated_images = network.decoder_model.predict(noise + np.repeat(b, repeated, axis=0),  batch_size=100)
            one_shot_generated_images = network.decoder_model.predict(noise + np.repeat(one_shot_b, repeated, axis=0), batch_size=100)
        elif 'productspace' in network_info['model_info']['network_class'].lower():
            generated_images = network.decoder_model.predict(np.concatenate([np.repeat(b, repeated, axis=0), noise], axis=-1),
                                                             batch_size=100)
            one_shot_generated_images = network.decoder_model.predict(np.concatenate([np.repeat(one_shot_b, repeated, axis=0), noise], axis=-1),
                                                                      batch_size=100)
        elif 'conditional' in network_info['model_info']['network_class'].lower():
            generated_images = network.decoder_model.predict([noise, to_categorical(gen_y_class, reader.get_n_label())], batch_size=100)
        else:
            generated_images = network.decoder_model.predict(noise, batch_size=100)

        numeric_dict = {}
        original_clssifier_loss, original_classifier_accuracy = mnist_classifier.evaluate(x, 
                                                                                          to_categorical(y, 
                                                                                                         num_classes=y_class.shape[0]), verbose=0)
        numeric_dict['original_accuracy'] = original_classifier_accuracy

        clssifier_loss, classifier_accuracy = mnist_classifier.evaluate(generated_images, 
                                                                        to_categorical(gen_y_class, num_classes=y_class.shape[0]), verbose=0)
        numeric_dict['accuracy'] = classifier_accuracy

        if 'oae' in network_info['model_info']['network_class'].lower():
            one_shot_clssifier_loss, one_shot_classifier_accuracy = mnist_classifier.evaluate(one_shot_generated_images, 
                                                                                              to_categorical(gen_y_class, 
                                                                                                             num_classes=y_class.shape[0]), 
                                                                                              verbose=0)
            numeric_dict['one_shot_accuracy'] = one_shot_classifier_accuracy

        origin_sharpness = np.min(network.blurr_model.predict(real_img, batch_size=100))
        gen_sharpness = np.min(network.blurr_model.predict(generated_images,batch_size=100))
        numeric_dict['sharpness_original'] = origin_sharpness
        numeric_dict['sharpness_gen'] = gen_sharpness
        numeric_dict['original_ssim_mean'] = np.mean([np.mean(get_ssim(real_img[y==cls])) for cls in y_class])
        numeric_dict['gen_ssim_mean'] = np.mean([np.mean(get_ssim(generated_images[gen_y_class==cls], 10)) for cls in y_class])

        if 'oae' in network_info['model_info']['network_class'].lower():
            one_shot_gen_sharpness = np.min(network.blurr_model.predict(one_shot_generated_images, batch_size=100))
            numeric_dict['sharpness_one_shot_gen'] = one_shot_gen_sharpness
            numeric_dict['one_shot_gen_ssim_mean'] = np.mean([np.mean(get_ssim(one_shot_generated_images[gen_y_class==cls], 10)) for cls in y_class])
        
        log.info(numeric_dict)

        tot_original_accuracy.append(numeric_dict['original_accuracy'])
        tot_original_ssim_mean.append(numeric_dict['original_ssim_mean'])
        tot_sharpness_original.append(numeric_dict['sharpness_original'])

        tot_accuracy.append(numeric_dict['accuracy'])
        tot_gen_ssim_mean.append(numeric_dict['gen_ssim_mean'])
        tot_sharpness_gen.append(numeric_dict['sharpness_gen'])

        if 'oae' in network_info['model_info']['network_class'].lower():
            tot_one_shot_accuracy.append(numeric_dict['one_shot_accuracy'])
            tot_one_shot_gen_ssim_mean.append(numeric_dict['one_shot_gen_ssim_mean'])
            tot_one_shot_sharpness_gen.append(numeric_dict['sharpness_one_shot_gen'])
    
    log.info('-----------------------------------------------------------------')
    log.info('Results of %s' % (model_aka))
    log.info('-----------------------------------------------------------------')
    log.info('Original accuracy: %.3f (\pm %.3f)' % (np.mean(tot_original_accuracy), np.std(tot_original_accuracy)))
    log.info('Original_ssim_mean: %.3f (\pm %.3f)' % (np.mean(tot_original_ssim_mean), np.std(tot_original_ssim_mean)))
    log.info('Original_sharpness: %.3f (\pm %.3f)' % (np.mean(tot_sharpness_original), np.std(tot_sharpness_original)))

    log.info('Accuracy: %.3f (\pm %.3f)' % (np.mean(tot_accuracy), np.std(tot_accuracy)))
    log.info('Ssim_mean: %.3f (\pm %.3f)' % (np.mean(tot_gen_ssim_mean), np.std(tot_gen_ssim_mean)))
    log.info('Sharpness: %.3f (\pm %.3f)' % (np.mean(tot_sharpness_gen), np.std(tot_sharpness_gen)))

    if 'oae' in network_info['model_info']['network_class'].lower():
        log.info('One-shot Accuracy: %.3f (\pm %.3f)' % (np.mean(tot_one_shot_accuracy), np.std(tot_one_shot_accuracy)))
        log.info('One-shot Ssim_mean: %.3f (\pm %.3f)' % (np.mean(tot_one_shot_gen_ssim_mean), np.std(tot_one_shot_gen_ssim_mean)))
        log.info('One-shot Sharpness: %.3f (\pm %.3f)' % (np.mean(tot_one_shot_sharpness_gen), np.std(tot_one_shot_sharpness_gen)))
#         tot_numerics = [
#             tot_original_accuracy,
#             tot_original_ssim_mean,
#             tot_sharpness_original,
#             tot_reconstruction,
#             tot_accuracy,
#             tot_gen_ssim_mean,
#             tot_sharpness_gen,
#             tot_one_shot_accuracy,
#             tot_one_shot_gen_ssim_mean,
#             tot_one_shot_sharpness_gen
#         ]
#     else:
#         tot_numerics = [
#             tot_original_accuracy,
#             tot_original_ssim_mean,
#             tot_sharpness_original,
#             tot_reconstruction,
#             tot_accuracy,
#             tot_gen_ssim_mean,
#             tot_sharpness_gen
#         ]


#     np.save('./analysis/numerics_%s.npy' % model_aka, np.array(tot_numerics))
    log.info('-----------------------------------------------------------------------------------')<jupyter_output><empty_output><jupyter_text>Model Performance<jupyter_code>model_path = 'mnist_pretrained/mnist_imbalance_psoae'
model_aka = 'PSOAE'
path_info_config = "configurations/mnist/psoae_path_info.cfg"
network_info_config = "configurations/mnist/psoae_network_info.cfg"
feature_b = False

get_numerics(model_path, model_aka, path_info_config, network_info_config, feature_b)<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
{'fraction_non_alphanumeric': 0.15705433746425168, 'fraction_numerical': 0.011141563393708294, 'mean_word_length': 2.9662098298676747, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '273625', 'license': 'MIT', 'n_tokens_mistral': 5438, 'n_tokens_neox': 5234, 'n_words': 729, 'path': 'mnist_numerics.ipynb', 'repo_name': 'Young-won/ornstein_auto_encoders', 'size': '16784'}