<commit_before>"""
This module contains an implementation of a runner that is supposed to be
used in case we train a language model. Instead of decoding sentences in
computes its perplexities given the decoder.
"""
#tests: lint

from neuralmonkey.learning_utils import feed_dicts

class PerplexityRunner(object):
    def __init__(self, decoder, batch_size):
        self.decoder = decoder
        self.batch_size = batch_size
        self.vocabulary = decoder.vocabulary

    def __call__(self, sess, dataset, coders):
        if not dataset.has_series(self.decoder.data_id):
            raise Exception("Dataset must have the target values ({})"
                            "for computing perplexity."
                            .format(self.decoder.data_id))

        batched_dataset = dataset.batch_dataset(self.batch_size)
        losses = [self.decoder.train_loss,
                  self.decoder.runtime_loss]
        perplexities = []

        train_loss = 0.0
        runtime_loss = 0.0

        batch_count = 0
        for batch in batched_dataset:
            batch_count += 1
            batch_feed_dict = feed_dicts(batch, coders, train=False)
            cross_entropies, opt_loss, dec_loss = sess.run(
                [self.decoder.cross_entropies] + losses,
                feed_dict=batch_feed_dict)

            perplexities.extend([2 ** xent for xent in cross_entropies])
            train_loss += opt_loss
            runtime_loss += dec_loss

        avg_train_loss = train_loss / batch_count
        avg_runtime_loss = runtime_loss / batch_count

        return perplexities, avg_train_loss, avg_runtime_loss
<commit_msg>make pylint even more happy<commit_after>"""
This module contains an implementation of a runner that is supposed to be
used in case we train a language model. Instead of decoding sentences in
computes its perplexities given the decoder.
"""
#tests: lint

from neuralmonkey.learning_utils import feed_dicts

#pylint: disable=too-few-public-methods
class PerplexityRunner(object):
    def __init__(self, decoder, batch_size):
        self.decoder = decoder
        self.batch_size = batch_size
        self.vocabulary = decoder.vocabulary

    def __call__(self, sess, dataset, coders):
        if not dataset.has_series(self.decoder.data_id):
            raise Exception("Dataset must have the target values ({})"
                            "for computing perplexity."
                            .format(self.decoder.data_id))
        perplexities = []
        train_loss = 0.0
        runtime_loss = 0.0
        batch_count = 0

        for batch in dataset.batch_dataset(self.batch_size):
            batch_count += 1
            batch_feed_dict = feed_dicts(batch, coders, train=False)
            cross_entropies, opt_loss, dec_loss = sess.run(
                [self.decoder.cross_entropies,
                 self.decoder.train_loss,
                 self.decoder.runtime_loss],
                feed_dict=batch_feed_dict)

            perplexities.extend([2 ** xent for xent in cross_entropies])
            train_loss += opt_loss
            runtime_loss += dec_loss

        avg_train_loss = train_loss / batch_count
        avg_runtime_loss = runtime_loss / batch_count

        return perplexities, avg_train_loss, avg_runtime_loss

{'fraction_non_alphanumeric': 0.07188547060615291, 'fraction_numerical': 0.0042643923240938165, 'mean_word_length': 2.3544433094994894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2388929', 'n_tokens_mistral': 959, 'n_tokens_neox': 899, 'n_words': 240}