<commit_before>"""
Evaluates a model trained by `trainMLP`.

TODO(andrei): Integrate with training script for convenience.
"""
try:
    # Keep this import on top!
    from spacy.en import English
except:
    # Shit, son.
    raise

import click
from os.path import join as pjoin
import sys
import argparse
from keras.models import model_from_json

import numpy as np
import scipy.io
from sklearn.externals import joblib

from features import get_questions_matrix_sum, get_images_matrix, \
    get_answers_matrix
from utils import grouper, lines


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-model', type=str, required=True,
                        help="JSON dump of saved model structure.")
    parser.add_argument('-weights', type=str, required=True,
                        help="Saved weights (checkpoint).")
    parser.add_argument('-results', type=str, required=True,
                        help="File where to write the results.")
    parser.add_argument('-results_json', type=str, required=True,
                        help="File where to dump the evaluation results in "
                             "JSON format, so that the official VQA toolkit "
                             "can read it.")
    parser.add_argument('-dataroot', type=str, default='/data/vqa')
    args = parser.parse_args()
    root = args.dataroot

    model = model_from_json(open(args.model).read())
    model.load_weights(args.weights)
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    questions_val = lines(pjoin(root, 'Preprocessed', 'questions_val2014.txt'))
    questions_id = lines(pjoin(root, 'Preprocessed', 'questions_id_val2014.txt'))
    answers_val = lines(pjoin(root, 'Preprocessed', 'answers_val2014_all.txt'))
    images_val = lines(pjoin(root, 'Preprocessed', 'images_val2014_all.txt'))
    vgg_model_path = pjoin(root, 'coco', 'vgg_feats.mat')

    print('Model compiled, weights loaded...')
    # TODO(andrei): If this fails, use pickle directly.
    labelencoder = joblib.load(pjoin(root, 'models', 'labelencoder.pkl'))

    features_struct = scipy.io.loadmat(vgg_model_path)
    VGGfeatures = features_struct['feats']
    print('loaded vgg features')
    image_ids = lines(pjoin(root, 'coco_vgg_IDMap.txt'))

    img_map = {}
    for ids in image_ids:
        id_split = ids.split()
        img_map[id_split[0]] = int(id_split[1])

    nlp = English()
    print('loaded word2vec features')

    nb_classes = 1000
    y_predict_text = []

    # TODO(andrei): Configure this via args.
    batchSize = 512

    stuff = list(zip(
        grouper(questions_val, batchSize, fillvalue=questions_val[0]),
        grouper(answers_val, batchSize, fillvalue=answers_val[0]),
        grouper(images_val, batchSize, fillvalue=images_val[0])))

    # TODO(andrei): Consider doing this in parallel, and on CPU, as it *may*
    # be faster.
    with click.progressbar(stuff) as pbar:
        for (qu_batch, an_batch, im_batch) in pbar:
            X_q_batch = get_questions_matrix_sum(qu_batch, nlp)
            if 'language_only' in args.model:
                X_batch = X_q_batch
            else:
                X_i_batch = get_images_matrix(im_batch, img_map, VGGfeatures)
                X_batch = np.hstack((X_q_batch, X_i_batch))

            y_predict = model.predict_classes(X_batch, verbose=0)
            y_predict_text.extend(labelencoder.inverse_transform(y_predict))

    correct_val = 0.0
    total = 0
    f1 = open(args.results, 'w')
    print("Will dump resulting answers in JSON format to file: [{0}]".format(
        args.results_json
    ))
    result_file_json = open(args.results_json, 'w')
    result_file_json.write("[")

    all_preds = list(zip(y_predict_text, answers_val, questions_val,
                        questions_id, images_val))
    for idx, (prediction, truth, question, question_id, image) in enumerate(all_preds):<commit_msg>quick bugfix<commit_after>"""
Evaluates a model trained by `trainMLP`.

TODO(andrei): Integrate with training script for convenience.
"""
try:
    # Keep this import on top!
    from spacy.en import English
except:
    # Shit, son.
    raise

import click
from os.path import join as pjoin
import argparse
from keras.models import model_from_json

import scipy.io
from sklearn.externals import joblib

from features import get_questions_matrix_sum, get_images_matrix, \
    get_answers_matrix
from utils import grouper, lines


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-model', type=str, required=True,
                        help="JSON dump of saved model structure.")
    parser.add_argument('-weights', type=str, required=True,
                        help="Saved weights (checkpoint).")
    parser.add_argument('-results', type=str, required=True,
                        help="File where to write the results.")
    parser.add_argument('-results_json', type=str, required=True,
                        help="File where to dump the evaluation results in "
                             "JSON format, so that the official VQA toolkit "
                             "can read it.")
    parser.add_argument('-dataroot', type=str, default='/data/vqa')
    args = parser.parse_args()
    root = args.dataroot

    model = model_from_json(open(args.model).read())
    model.load_weights(args.weights)
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

    questions_val = lines(pjoin(root, 'Preprocessed', 'questions_val2014.txt'))
    questions_id = lines(pjoin(root, 'Preprocessed', 'questions_id_val2014.txt'))
    answers_val = lines(pjoin(root, 'Preprocessed', 'answers_val2014_all.txt'))
    images_val = lines(pjoin(root, 'Preprocessed', 'images_val2014_all.txt'))
    vgg_model_path = pjoin(root, 'coco', 'vgg_feats.mat')

    print('Model compiled, weights loaded...')
    # TODO(andrei): If this fails, use pickle directly.
    labelencoder = joblib.load(pjoin(root, 'models', 'labelencoder.pkl'))

    features_struct = scipy.io.loadmat(vgg_model_path)
    VGGfeatures = features_struct['feats']
    print('loaded vgg features')
    image_ids = lines(pjoin(root, 'coco_vgg_IDMap.txt'))

    img_map = {}
    for ids in image_ids:
        id_split = ids.split()
        img_map[id_split[0]] = int(id_split[1])

    nlp = English()
    print('loaded word2vec features')

    nb_classes = 1000
    y_predict_text = []

    # TODO(andrei): Configure this via args.
    batchSize = 512

    stuff = list(zip(
        grouper(questions_val, batchSize, fillvalue=questions_val[0]),
        grouper(answers_val, batchSize, fillvalue=answers_val[0]),
        grouper(images_val, batchSize, fillvalue=images_val[0])))

    # TODO(andrei): Consider doing this in parallel, and on CPU, as it *may*
    # be faster.
    with click.progressbar(stuff) as pbar:
        for (qu_batch, an_batch, im_batch) in pbar:
            X_q_batch = get_questions_matrix_sum(qu_batch, nlp)
            X_i_batch = get_images_matrix(im_batch, img_map, VGGfeatures)
            # TODO(Bernhard): make this choose the right preprocessing and right model...
            y_predict = model.predict_classes([X_q_batch, X_i_batch], verbose=0)
            # TODO(Bernhard): verify that predict_classes sets dropout to 0
            y_predict_text.extend(labelencoder.inverse_transform(y_predict))

    correct_val = 0.0
    total = 0
    f1 = open(args.results, 'w')
    print("Will dump resulting answers in JSON format to file: [{0}]".format(
        args.results_json
    ))
    result_file_json = open(args.results_json, 'w')
    result_file_json.write("[")

    all_preds = list(zip(y_predict_text, answers_val, questions_val,
                        questions_id, images_val))
    for idx, (prediction, truth, question, question_id, image) in enumerate(all_preds):
{'fraction_non_alphanumeric': 0.10943299628823755, 'fraction_numerical': 0.009087418405222066, 'mean_word_length': 3.176376269374666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2815398', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2424, 'n_words': 581}