<commit_before>import logging

import numpy
from blocks.extensions.monitoring import SimpleExtension, MonitoringExtension
from blocks.initialization import NdarrayInitialization
from blocks.monitoring.evaluators import DatasetEvaluator
from blocks.serialization import secure_dump, dump_and_add_to_dump
from fuel.transformers import Transformer


logger = logging.getLogger(__name__)  # no idea what this is dude
SAVED_TO = "saved_to"


class PadAndAddMasks(Transformer):
    """
    If you don't know what transformers are or what they do or how they work, but would like to, either try finding
    something in the Fuel docs or ask me.
    """
    def __init(self, data_stream, **kwargs):
        if data_stream.axis_labels:
            kwargs.setdefault('axis_labels', data_stream.axis_labels.copy())
        super(PadAndAddMasks, self).__init__(data_stream, data_stream.produces_examples, **kwargs)

    @property
    def sources(self):
        """
        Because the transformer adds a source on top of what is in the dataset (namely the mask), we need to define
        this.
        """
        return "character_seqs", "seq_mask"

    def transform_batch(self, batch):
        maxlen = max(example.shape[0] for example in batch[0])
        # For one, zero-pad the sequences
        # Also build the mask
        # Note that, because the transformer doesn't do enough stuff yet, the sequences (batch[0]) are also transposed
        # to go from one-sequence-per-row (the way it's stored) to the weird Blocks RNN format (one-sequence-per-column)
        padded_seqs = numpy.zeros((batch[0].shape[0], maxlen), dtype="int32")
        # mask is int8 because apparently it's converted to float later, and higher ints would complain about loss of
        # precision
        mask = numpy.zeros((batch[0].shape[0], maxlen), dtype="int8")
        for (example_ind, example) in enumerate(batch[0]):
            # we go through the sequences and simply put them into the padded array; this will leave 0s wherever the
            # sequence is shorter than maxlen. similarly, mask will be set to 1 only up to the length of the respective
            # sequence
            # note that the transpose is done implicitly by essentially swapping indices
            padded_seqs[example_ind, :len(example)] = example
            mask[example_ind, :len(example)] = 1
        return padded_seqs, mask


class LSTMGateBias(NdarrayInitialization):
    """
    >>Might as well ignore this at the moment!<<

    Takes a normal initialization but biases the parts that corresponds to the gates by given values.

    The "forget_value" should be positive; the "inp_out_value" should be negative. Values could e.g. be something
    between 1-5 in absolute value. Supposedly, the precise values don't matter much..."""
    def __init__(self, init, forget_value=3, inp_out_value=-3):
        self.init = init
        self.forget_value = forget_value
        self.inp_out_value = inp_out_value

    def generate(self, rng, shape):
        weights = self.init.generate(rng, shape)
        # len(weights) is bound to be divisible by 4, else you did something wrong
        weights[(len(weights)/4):(len(weights)/2)] += self.forget_value
        weights[:(len(weights)/4)] += self.inp_out_value
        weights[(3*len(weights)/4):] += self.inp_out_value
        return weights


class EarlyStopping(SimpleExtension, MonitoringExtension):
    """
    Extension that tracks some quantity (PLEASE use cost on the validation set here!) and creates a checkpoint whenever
    a new best-value-so-far has been achieved. In addition, training is stopped if performance hasn't improved since n
    epochs. Note: "Improvement" is not needed over the current best value, but only over the last time we measured.<commit_msg>PadAndAddMasks transformer uses general padding+masking function.<commit_after>import logging

import numpy
from blocks.extensions.monitoring import SimpleExtension, MonitoringExtension
from blocks.initialization import NdarrayInitialization
from blocks.monitoring.evaluators import DatasetEvaluator
from blocks.serialization import secure_dump, dump_and_add_to_dump
from fuel.transformers import Transformer

from util import pad_mask


logger = logging.getLogger(__name__)  # no idea what this is dude
SAVED_TO = "saved_to"


class PadAndAddMasks(Transformer):
    """
    If you don't know what transformers are or what they do or how they work, but would like to, either try finding
    something in the Fuel docs or ask me.
    """
    def __init(self, data_stream, **kwargs):
        if data_stream.axis_labels:
            kwargs.setdefault('axis_labels', data_stream.axis_labels.copy())
        super(PadAndAddMasks, self).__init__(data_stream, data_stream.produces_examples, **kwargs)

    @property
    def sources(self):
        """
        Because the transformer adds a source on top of what is in the dataset (namely the mask), we need to define
        this.
        """
        return "character_seqs", "seq_mask"

    def transform_batch(self, batch_sources):
        """Implements transformer hook to add padding and masks to datastream.

        batch_sources is a tuple containing all the sources from datastream.
        In our case it's just one source, so we handle it accordingly.
        """
        return pad_mask(batch_sources[0])


class LSTMGateBias(NdarrayInitialization):
    """
    >>Might as well ignore this at the moment!<<

    Takes a normal initialization but biases the parts that corresponds to the gates by given values.

    The "forget_value" should be positive; the "inp_out_value" should be negative. Values could e.g. be something
    between 1-5 in absolute value. Supposedly, the precise values don't matter much..."""
    def __init__(self, init, forget_value=3, inp_out_value=-3):
        self.init = init
        self.forget_value = forget_value
        self.inp_out_value = inp_out_value

    def generate(self, rng, shape):
        weights = self.init.generate(rng, shape)
        # len(weights) is bound to be divisible by 4, else you did something wrong
        weights[(len(weights)/4):(len(weights)/2)] += self.forget_value
        weights[:(len(weights)/4)] += self.inp_out_value
        weights[(3*len(weights)/4):] += self.inp_out_value
        return weights


class EarlyStopping(SimpleExtension, MonitoringExtension):
    """
    Extension that tracks some quantity (PLEASE use cost on the validation set here!) and creates a checkpoint whenever
    a new best-value-so-far has been achieved. In addition, training is stopped if performance hasn't improved since n
    epochs. Note: "Improvement" is not needed over the current best value, but only over the last time we measured.
{'fraction_non_alphanumeric': 0.07855657855657856, 'fraction_numerical': 0.005346005346005346, 'mean_word_length': 3.433838051349572, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '181247', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1821, 'n_words': 700}