<commit_before>from __future__ import print_function

import numpy as np
import h5py
import theano

import argparse
import itertools
import subprocess
import logging
import time
import codecs
import os
from copy import deepcopy
import math
import sys

from data_generator import VisualWordDataGenerator
import models

# Set up logger
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
logger = logging.getLogger(__name__)

# Dimensionality of image feature vector
IMG_FEATS = 4096


class GroundedTranslationGenerator:

    def __init__(self, args):
        self.args = args
        self.vocab = dict()
        self.unkdict = dict()
        self.counter = 0
        self.maxSeqLen = 0

        # consistent with models.py
        self.use_sourcelang = args.source_vectors is not None
        self.use_image = not args.no_image

        # this results in two file handlers for dataset (here and
        # data_generator)
        if not self.args.dataset:
            logger.warn("No dataset given, using flickr8k")
            self.dataset = h5py.File("flickr8k/dataset.h5", "r")
        else:
            self.dataset = h5py.File("%s/dataset.h5" % self.args.dataset, "r")

        if self.args.debug:
            theano.config.optimizer = 'None'
            theano.config.exception_verbosity = 'high'

    def generationModel(self):
        '''
        In the model, we will merge the VGG image representation with
        the word embeddings. We need to feed the data as a list, in which
        the order of the elements in the list is _crucial_.
        '''

        self.data_gen = VisualWordDataGenerator(self.args,
                                                self.args.dataset)
        self.args.checkpoint = self.find_best_checkpoint()
        self.data_gen.set_vocabulary(self.args.checkpoint)
        self.vocab_len = len(self.data_gen.index2word)
        self.index2word = self.data_gen.index2word
        self.word2index = self.data_gen.word2index

        if self.use_sourcelang:
            # HACK FIXME unexpected problem with input_data
            self.hsn_size = 256
        else:
            self.hsn_size = 0

        m = models.OneLayerLSTM(self.args.hidden_size, self.vocab_len,
                                self.args.dropin,
                                self.args.optimiser, self.args.l2reg,
                                hsn_size=self.hsn_size,
                                weights=self.args.checkpoint,
                                gru=self.args.gru)

        self.model = m.buildKerasModel(use_sourcelang=self.use_sourcelang,
                                       use_image=self.use_image)

        self.generate_sentences(self.args.checkpoint, val=not self.args.test)
        self.bleu_score(self.args.checkpoint, val=not self.args.test)
        self.calculate_pplx(self.args.checkpoint, val=not self.args.test)

    def generate_sentences(self, filepath, val=True):
        """
        Generates descriptions of images for --generation_timesteps
        iterations through the LSTM. Each input description is clipped to
        the first <BOS> token, or, if --generate_from_N_words is set, to the
        first N following words (N + 1 BOS token).
        This process can be additionally conditioned
        on source language hidden representations, if provided by the
        --source_vectors parameter.
        The output is clipped to the first EOS generated, if it exists.

        TODO: beam search
        TODO: duplicated method with generate.py
        """
        prefix = "val" if val else "test"
        handle = codecs.open("%s/%sGenerated" % (filepath, prefix), "w",
                             'utf-8')
        logger.info("Generating %s descriptions", prefix)

        start_gen = self.args.generate_from_N_words  # Default 0
        start_gen = start_gen + 1  # include BOS

        # prepare the datastructures for generation (no batching over val)
        arrays = self.make_generation_arrays(prefix, start_gen,
                 generation=self.args.use_predicted_tokens)
        N_sents = arrays[0].shape[0]
        logger.debug("Input arrays %d", len(arrays))
        logger.debug("Instances %d", len(arrays[0]))

        # complete_sentences = [["<S>"] for _ in range(N_sents)]

        complete_sentences = [[] for _ in range(N_sents)]
        for t in range(start_gen):  # minimum 1
            for i in range(N_sents):
                w = np.argmax(arrays[0][i, t])

        logger.debug(complete_sentences[3])
        logger.debug(self.index2word[np.argmax(arrays[0][0])])

        for t in range(start_gen, self.args.generation_timesteps):
            # we take a view of the datastructures, which means we're only
            # ever generating a prediction for the next word. This saves a
            # lot of cycles.
            preds = self.model.predict([arr[:, 0:t] for arr in arrays],
                                       verbose=0)

            # Look at the last indices for the words.
            next_word_indices = np.argmax(preds[:, -1], axis=1)
            # update array[0]/sentence-so-far with generated words.
            for i in range(N_sents):
                arrays[0][i, t, next_word_indices[i]] = 1.
            next_words = [self.index2word[x] for x in next_word_indices]
            for i in range(len(next_words)):
                complete_sentences[i].append(next_words[i])

        # save each sentence until it hits the first end-of-string token
        for s in complete_sentences:
            handle.write(' '.join([x for x
                                   in itertools.takewhile(
                                       lambda n: n != "<E>", s[1:])]) + "\n")

        handle.close()

    def find_best_checkpoint(self):
        '''
        Read the summary file from the directory and scrape out the run ID of
        the highest BLEU scoring checkpoint. Then do an ls-stlye function in
        the directory and return the exact path to the best model.

        Assumes only one matching prefix in the model checkpoints directory.
        '''

        summary_data = open("%s/summary" % self.args.model_checkpoints).readlines()
        summary_data = [x.replace("\n", "") for x in summary_data]
        best_id = None
        target = "Best PPLX" if self.args.best_pplx else "Best BLEU"
        for line in summary_data:
            if line.startswith(target):
                best_id = "%03d" % (int(line.split(":")[1].split("|")[0]))<commit_msg>Fixed a bug where generation outside of training clipped the first word in a description. Thanks @ehasler!<commit_after>from __future__ import print_function

import numpy as np
import h5py
import theano

import argparse
import itertools
import subprocess
import logging
import time
import codecs
import os
from copy import deepcopy
import math
import sys

from data_generator import VisualWordDataGenerator
import models

# Set up logger
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
logger = logging.getLogger(__name__)

# Dimensionality of image feature vector
IMG_FEATS = 4096


class GroundedTranslationGenerator:

    def __init__(self, args):
        self.args = args
        self.vocab = dict()
        self.unkdict = dict()
        self.counter = 0
        self.maxSeqLen = 0

        # consistent with models.py
        self.use_sourcelang = args.source_vectors is not None
        self.use_image = not args.no_image

        # this results in two file handlers for dataset (here and
        # data_generator)
        if not self.args.dataset:
            logger.warn("No dataset given, using flickr8k")
            self.dataset = h5py.File("flickr8k/dataset.h5", "r")
        else:
            self.dataset = h5py.File("%s/dataset.h5" % self.args.dataset, "r")

        if self.args.debug:
            theano.config.optimizer = 'None'
            theano.config.exception_verbosity = 'high'

    def generationModel(self):
        '''
        In the model, we will merge the VGG image representation with
        the word embeddings. We need to feed the data as a list, in which
        the order of the elements in the list is _crucial_.
        '''

        self.data_gen = VisualWordDataGenerator(self.args,
                                                self.args.dataset)
        self.args.checkpoint = self.find_best_checkpoint()
        self.data_gen.set_vocabulary(self.args.checkpoint)
        self.vocab_len = len(self.data_gen.index2word)
        self.index2word = self.data_gen.index2word
        self.word2index = self.data_gen.word2index

        if self.use_sourcelang:
            # HACK FIXME unexpected problem with input_data
            self.hsn_size = 256
        else:
            self.hsn_size = 0

        m = models.OneLayerLSTM(self.args.hidden_size, self.vocab_len,
                                self.args.dropin,
                                self.args.optimiser, self.args.l2reg,
                                hsn_size=self.hsn_size,
                                weights=self.args.checkpoint,
                                gru=self.args.gru)

        self.model = m.buildKerasModel(use_sourcelang=self.use_sourcelang,
                                       use_image=self.use_image)

        self.generate_sentences(self.args.checkpoint, val=not self.args.test)
        self.bleu_score(self.args.checkpoint, val=not self.args.test)
        self.calculate_pplx(self.args.checkpoint, val=not self.args.test)

    def generate_sentences(self, filepath, val=True):
        """
        Generates descriptions of images for --generation_timesteps
        iterations through the LSTM. Each input description is clipped to
        the first <BOS> token, or, if --generate_from_N_words is set, to the
        first N following words (N + 1 BOS token).
        This process can be additionally conditioned
        on source language hidden representations, if provided by the
        --source_vectors parameter.
        The output is clipped to the first EOS generated, if it exists.

        TODO: beam search
        TODO: duplicated method with generate.py
        """
        prefix = "val" if val else "test"
        handle = codecs.open("%s/%sGenerated" % (filepath, prefix), "w",
                             'utf-8')
        logger.info("Generating %s descriptions", prefix)

        start_gen = self.args.generate_from_N_words  # Default 0
        start_gen = start_gen + 1  # include BOS

        # prepare the datastructures for generation (no batching over val)
        arrays = self.make_generation_arrays(prefix, start_gen,
                 generation=self.args.use_predicted_tokens)
        N_sents = arrays[0].shape[0]
        logger.debug("Input arrays %d", len(arrays))
        logger.debug("Instances %d", len(arrays[0]))

        # complete_sentences = [["<S>"] for _ in range(N_sents)]

        complete_sentences = [[] for _ in range(N_sents)]
        for t in range(start_gen):  # minimum 1
            for i in range(N_sents):
                w = np.argmax(arrays[0][i, t])

        logger.debug(complete_sentences[3])
        logger.debug(self.index2word[np.argmax(arrays[0][0])])

        for t in range(start_gen, self.args.generation_timesteps):
            # we take a view of the datastructures, which means we're only
            # ever generating a prediction for the next word. This saves a
            # lot of cycles.
            preds = self.model.predict([arr[:, 0:t] for arr in arrays],
                                       verbose=0)

            # Look at the last indices for the words.
            next_word_indices = np.argmax(preds[:, -1], axis=1)
            # update array[0]/sentence-so-far with generated words.
            for i in range(N_sents):
                arrays[0][i, t, next_word_indices[i]] = 1.
            next_words = [self.index2word[x] for x in next_word_indices]
            for i in range(len(next_words)):
                complete_sentences[i].append(next_words[i])

        # save each sentence until it hits the first end-of-string token
        for s in complete_sentences:
            handle.write(' '.join([x for x
                                   in itertools.takewhile(
                                       lambda n: n != "<E>", s)]) + "\n")

        handle.close()

    def find_best_checkpoint(self):
        '''
        Read the summary file from the directory and scrape out the run ID of
        the highest BLEU scoring checkpoint. Then do an ls-stlye function in
        the directory and return the exact path to the best model.

        Assumes only one matching prefix in the model checkpoints directory.
        '''

        summary_data = open("%s/summary" % self.args.model_checkpoints).readlines()
        summary_data = [x.replace("\n", "") for x in summary_data]
        best_id = None
        target = "Best PPLX" if self.args.best_pplx else "Best BLEU"
        for line in summary_data:
            if line.startswith(target):
                best_id = "%03d" % (int(line.split(":")[1].split("|")[0]))
{'fraction_non_alphanumeric': 0.08817881557723327, 'fraction_numerical': 0.007450649051386435, 'mean_word_length': 2.075106282475201, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2517111', 'n_tokens_mistral': 3798, 'n_tokens_neox': 3647, 'n_words': 1055}