<jupyter_start><jupyter_text>Evaluation  for MsPacman<jupyter_code>%matplotlib inline

import os
import sys
sys.path.append(os.path.expanduser("~/libs"))

import cv2
import numpy as np
import tensorflow as tf
import tensortools as tt

from model.frame_prediction import LSTMConv2DPredictionModel<jupyter_output><empty_output><jupyter_text>Hyperparams<jupyter_code># Data
INPUT_SEQ_LENGTH = 8
OUTPUT_SEQ_LENGTH = 8

# Evaluation
EVAL_BATCH_SIZE = 50
# validation while training
OUT_DIR_NAME = "out-eval"
NUM_SAMPLES = 10
GIF_FPS = 5<jupyter_output><empty_output><jupyter_text>Directory Paths:<jupyter_code>ROOT_DIR = "/work/sauterme/"
DATA_DIR = ROOT_DIR + "data"
TRAIN_DIR = ROOT_DIR + "train/pac/ss/2l3i5hp/c326464k533s121bn/wd1e-05/mse/QQ"

assert os.path.exists(TRAIN_DIR)<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>dataset_valid = tt.datasets.ms_pacman.MsPacmanValidDataset(DATA_DIR,
                                                           input_seq_length=INPUT_SEQ_LENGTH,
                                                           target_seq_length=OUTPUT_SEQ_LENGTH,
                                                           crop_size=[32, 32])
dataset_test = tt.datasets.ms_pacman.MsPacmanTestDataset(DATA_DIR,
                                                         input_seq_length=INPUT_SEQ_LENGTH,
                                                         target_seq_length=OUTPUT_SEQ_LENGTH,
                                                         crop_size=[32, 32],
                                                         skip_less_movement=False)<jupyter_output>File Ms_Pacman.zip has already been extracted.
File Ms_Pacman.zip has already been extracted.<jupyter_text>Runtime<jupyter_code>GPU_ID = 7
runtime = tt.core.DefaultRuntime(train_dir=TRAIN_DIR, gpu_devices=[GPU_ID])
runtime.list_checkpoints()
CHECKPOINT = tt.core.LATEST_CHECKPOINT
runtime.register_datasets(None, dataset_valid, dataset_test)
runtime.register_model(LSTMConv2DPredictionModel(main_loss=tt.loss.bce))<jupyter_output><empty_output><jupyter_text>Consider to restore the **EMA variables** as well when building the model. These might generate worse results in models using batch-normalization, since the shaddow variables might get restored properly...<jupyter_code>runtime.build(restore_checkpoint=CHECKPOINT, restore_model_params=True,
              restore_ema_variables=False, eval_mode=True, verbose=True)<jupyter_output>Restoring model parameters...

>>> Model:
alpha_ssim_loss   ->  0.0
output_activation  ->  TANH
lstm_cell_clip    ->  None
bias_init         ->  0.0
alpha_main_loss   ->  1.0
alpha_gdl_loss    ->  1.0
lstm_ksize_hidden  ->  (5, 5)
lstm_bn_hidden_hidden  ->  False
lstm_ksize_input  ->  (3, 3)
ksizes            ->  [(5, 5), (3, 3), (3, 3)]
lstm_use_peepholes  ->  True
lstm_bn_input_hidden  ->  False
strides           ->  [(1, 1), (2, 2), (1, 1)]
filters           ->  [32, 64, 64]
lstm_bn_peepholes  ->  False
bn_feature_enc    ->  True
main_loss         ->  MSE
scheduled_sampling_decay_rate  ->  1000.0
bn_feature_dec    ->  True
weight_decay      ->  1e-05
lstm_layers       ->  2

Found 144 update ops.
Selected checkpoint file: /work/sauterme/train/pac/ss/2l3i5hp/c326464k533s121bn/wd1e-05/mse/QQ/model.ckpt-100000
Restoring variables...

--------------------------------------------------------------------------------
conv-encoder/conv1/W:0                                                |  [...]<jupyter_text>Evaluation<jupyter_code>runtime.validate(EVAL_BATCH_SIZE)
runtime.test(EVAL_BATCH_SIZE)<jupyter_output>@100000: Starting test (batch-size: 50, dataset-size: 13056):
100%▕██████████▏- 754s - loss: 0.0066 - psnr: 50.1164 - ssim9: 0.9828 - f1-mae: 0.0025 - f1-ssim5: 0.9893 - sharpdiff: 29.9851 - bce: 0.3785 - ssim7: 0.9828 - f2-mae: 0.0031 - ssim: 0.9829 - f2-psnr: 50.5941 - f2-mse: 9.0715e-04 - ssim5: 0.9833 - msssim: 0.9844 - f1-mse: 6.2366e-04 - mae: 0.0040 - rsse: 3.3069 - f1-sharpdiff: 30.0961 - f1-psnr: 50.5197 - mse: 0.0016 - f2-sharpdiff: 30.1816 - f2-ssim5: 0.9860<jupyter_text>VisualizationsOn which dataset we want to test on on the next section.<jupyter_code>dataset_eval = dataset_test<jupyter_output><empty_output><jupyter_text>Random predictionEither as **binary** (like in training) or **float** (as in raw dataset)...<jupyter_code>def write_animation(dir_path, inputs, targets, predictions, fps):
    concat_tgt = np.concatenate((inputs, targets))
    concat_pred = np.concatenate((inputs, predictions))

    tt.utils.video.write_multi_gif(os.path.join(dir_path, "anim-{:02d}.gif".format(i)),
                                   [concat_tgt, concat_pred],
                                   fps=fps, pad_value=1.0)

    tt.utils.video.write_multi_image_sequence(os.path.join(dir_path, "timeline-{:02d}.png".format(i)),
                                              [concat_tgt, concat_pred],
                                              pad_value=1.0)

def show(inputs, targets, predictions, rows=2):
    tt.visualization.display_batch(inputs, ncols=5, nrows=rows, title="Inputs")
    tt.visualization.display_batch(targets, ncols=5, nrows=rows, title="Targets")
    tt.visualization.display_batch(predictions, ncols=5, nrows=rows, title="Predictions")
dir_path = os.path.join(runtime.train_dir, OUT_DIR_NAME, "random")

inputs, targets = dataset_eval.get_batch(NUM_SAMPLES)

predictions = runtime.predict(inputs)

show(inputs[0], targets[0], predictions[0])
for i in range(inputs.shape[0]):
    write_animation(dir_path, inputs[i], targets[i], predictions[i], GIF_FPS)<jupyter_output><empty_output><jupyter_text>Specific Predictions TBD !!!We are using the inputs used in _Gitgub: Adversarial Video Generation_ cropped out of the website. These consist of two sequences...<jupyter_code>SOURCE_PATH = "assets/other/pacman/"
def read_sequence(dir_path, seq_id):
    image_list = []
    for i in range(INPUT_SEQ_LENGTH+OUTPUT_SEQ_LENGTH):
        image_path = os.path.join(dir_path, str(seq_id), "{:02d}.png".format(i))
        image = tt.utils.image.read(image_path, color_flags = cv2.IMREAD_GRAYSCALE)
        image_list.append(image)
    seq = np.array(image_list)
    seq = seq / 255.0
    seq = np.expand_dims(seq, 0)
    return seq[:,:INPUT_SEQ_LENGTH] , seq[:,INPUT_SEQ_LENGTH:] 
dir_path = os.path.join(runtime.train_dir, OUT_DIR_NAME, "spec")

for i in range(6):
    inputs, targets = read_sequence(SOURCE_PATH, i)
    predictions = runtime.predict(inputs)
    
    show(inputs[0], targets[0], predictions[0])
    write_animation(dir_path, inputs[0], targets[0], predictions[0], GIF_FPS)<jupyter_output><empty_output><jupyter_text>Fulll ImageIs the FCN model worth it? :)<jupyter_code>dataset_valid = tt.datasets.ms_pacman.MsPacmanValidDataset(DATA_DIR,
                                                           input_seq_length=INPUT_SEQ_LENGTH,
                                                           target_seq_length=OUTPUT_SEQ_LENGTH,
                                                           skip_less_movement=False,
                                                           random_flip=False)
dataset_test = tt.datasets.ms_pacman.MsPacmanTestDataset(DATA_DIR,
                                                         input_seq_length=INPUT_SEQ_LENGTH,
                                                         target_seq_length=OUTPUT_SEQ_LENGTH,
                                                         skip_less_movement=False,
                                                         random_flip=False)
dataset_eval = dataset_test
runtime.register_datasets(None, dataset_valid, dataset_test)
runtime.build(restore_checkpoint=CHECKPOINT, restore_model_params=True,
              restore_ema_variables=False, eval_mode=True)
dir_path = os.path.join(runtime.train_dir, OUT_DIR_NAME, "full")

inputs, targets = dataset_eval.get_batch(NUM_SAMPLES)

predictions = runtime.predict(inputs)

show(inputs[0], targets[0], predictions[0])
for i in range(inputs.shape[0]):
    write_animation(dir_path, inputs[i], targets[i], predictions[i], GIF_FPS)<jupyter_output><empty_output><jupyter_text>Full Image - Longer Time Range<jupyter_code># Data
INPUT_SEQ_LENGTH = 8
OUTPUT_SEQ_LENGTH = 24

dataset_valid = tt.datasets.ms_pacman.MsPacmanValidDataset(DATA_DIR,
                                                           input_seq_length=INPUT_SEQ_LENGTH,
                                                           target_seq_length=OUTPUT_SEQ_LENGTH,
                                                           skip_less_movement=False,
                                                           random_flip=False)
dataset_test = tt.datasets.ms_pacman.MsPacmanTestDataset(DATA_DIR,
                                                         input_seq_length=INPUT_SEQ_LENGTH,
                                                         target_seq_length=OUTPUT_SEQ_LENGTH,
                                                         skip_less_movement=False,
                                                         random_flip=False)
dataset_eval = dataset_test
runtime.register_datasets(None, dataset_valid, dataset_test)
runtime.build(restore_checkpoint=CHECKPOINT, restore_model_params=True,
              restore_ema_variables=False, eval_mode=True)
dir_path = os.path.join(runtime.train_dir, OUT_DIR_NAME, "full-long")

inputs, targets = dataset_eval.get_batch(NUM_SAMPLES)

predictions = runtime.predict(inputs)

show(inputs[0], targets[0], predictions[0])
for i in range(inputs.shape[0]):
    write_animation(dir_path, inputs[i], targets[i], predictions[i], GIF_FPS)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10822375286996452, 'fraction_numerical': 0.03141306616572741, 'mean_word_length': 2.4834605597964376, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 56, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '82529', 'license': 'MIT', 'n_tokens_mistral': 3195, 'n_tokens_neox': 2959, 'n_words': 501, 'path': 'src/pac_eval.ipynb', 'repo_name': 'bsautermeister/imseq', 'size': '9582'}