<commit_before>from blocks.bricks.recurrent import BaseRecurrent, recurrent
from blocks.bricks import Initializable, Linear
from blocks.bricks.base import application
from blocks.bricks.parallel import Fork

class RecurrentStack(BaseRecurrent, Initializable):
    u"""Stack of recurrent networks.

    Build a stack of recurrent layers from a supplied list of
    BaseRecurrent objects. Each object must have a `sequences`,
    `contexts`, `states` and `outputs` parameters to its `apply` method.
    It is assumed that the `sequences` of all objects has an "inputs"
    element. It is assumed that all `states` have a "states" element
    (this can be configured with `states_name` parameter.)

    The `sequences` (the "inputs") of the the first layer (layer 0) is
    used as the `sequence` of the stack (the instance of this class.)
    The "mask" element of the `sequences` is carried to all layers that
    have a "mask" element in their `sequences`.
    For all the following layers, all elements of the `sequences`, except
    for "mask", are forked from the "states" element of the of the
    `states` of the previous layer.

    The concatenated `states` of all layers is the `states` of the stack.
    In order to avoid conflict, the names of the elements appearing in the
    `states` of each layers are suffixed with a '_<n>' before combining
    them. The '<n>' is the number of the layer starting from 0.
    The same process is true for the `contexts` and `outputs` of the
    stack.

    Parameters
    ----------
    transitions : list
        List of recurrent units to use in each layer.
    prototype : :class:`.FeedForward`, optional
        The transformation applied to each states of each layer  when it is
         forked to the sequences of the next layer. By
        default a :class:`.Linear` transformation with biases is used.
    states_name : string
        In a stack of RNN the state of each layer is used as input to the
        next. This string identify which element in the output of each
        layer should be used for this task. By default the element is
        called "states". To be more precise, this is the name of the
        element in the outputs of the apply method of each transition
        (layer) that is used, via fork, as the sequences (input) of the
        next layer. The same element should also appear in the states
        parameter of the apply method.
    sync : bool
        If true, the input of each layer is forked from the states of the
        lower layer from the previous step of the RNN time sequence.
        If false (async, default) the input is forked from the states
        output of the same step.
        Looking at other examples, e.g., it looks like async is the
        correct way.
       https://github.com/karpathy/char-rnn/blob/master/model/LSTM.lua#L25

    Notes
    -----
    See :class:`.BaseRecurrent` for more initialization parameters.

    """
    def __init__(self, transitions, prototype=None, sync=False,
                 states_name="states", **kwargs):
        super(RecurrentStack, self).__init__(**kwargs)

        self.sync = sync
        self.states_name = states_name

        for d, transition in enumerate(transitions):
            transition.name += '_' + str(d)
        self.transitions = transitions

        if prototype is None:
            # By default use Linear (with bias) as a default fork.
            # This overrides the bad default inside Fork which is without bias.
            # Yes I know use_bias=True us the default, but it is a big deal so
            # I write it down explicitly.
            prototype = Linear(use_bias=True)
        depth = len(transitions)
        self.forks = [Fork(self.normal_inputs(d), prototype=prototype)
                      for d in range(1, depth)]

        self.children = self.transitions + self.forks

        # Programmatically fix the parameters of the  recurrent decoration
        setattr(self.apply, 'sequences', transitions[0].apply.sequences)
        for t in ["states", "contexts", "outputs"]:
            v = [s + '_' + str(d)
                 for d, transition in enumerate(transitions)
                 for s in getattr(transition.apply, t)]
            setattr(self.apply, t, v)

    def normal_inputs(self, d):
        transition = self.transitions[d]
        return [name for name in transition.apply.sequences
                if 'mask' not in name]

    def _push_allocation_config(self):
        # Configure the forks that connect the "states" element in the `states`
        # of one layer to the elements in the `sequences` of the next layer,
        # excluding "mask".
        # This involves `get_dim` requests
        # to the transitions. To make sure that it answers
        # correctly we should finish its configuration first.
        for transition in self.transitions:
            transition.push_allocation_config()

        for d, fork in enumerate(self.forks):
            fork.input_dim = self.transitions[d].get_dim(self.states_name)
            fork.output_dims = self.transitions[d+1].get_dims(
                fork.output_names)

    @recurrent(sequences=[], states=[], outputs=[], contexts=[])
    def apply(self, **kwargs):
        """Apply the stack of transitions.

        Parameters
        ----------
        All parameters are of type :class:`~tensor.TensorVariable`.

        Returns
        -------
        All return values are of type :class:`~tensor.TensorVariable`.

        """
        results = []
        for d, transition in enumerate(self.transitions):
            sequences_names = transition.apply.sequences
            if d == 0:
                layer_kwargs = dict(
                    (s, kwargs.get(s)) for s in sequences_names)
            else:
                if self.sync:
                    last_states = kwargs[self.states_name + "_" + str(d-1)]
                layer_kwargs = dict(zip(
                    self.normal_inputs(d),
                    self.forks[d-1].apply(last_states, as_list=True)))
                if "mask" in sequences_names:
                    layer_kwargs["mask"] = kwargs.get("mask")

            for t in ["states", "contexts", "outputs"]:
                suffix = '_' + str(d)
                for s in getattr(transition.apply, t):
                    layer_kwargs[s] = kwargs.get(s + suffix)

            result = transition.apply(iterate=False, **layer_kwargs)
            results.extend(result)
            if not self.sync:
                i = transition.apply.outputs.index(self.states_name)
                last_states = result[i]

        return tuple(results)

    def get_dim(self, name):
        if name in self.transitions[0].apply.sequences:
            return self.transitions[0].get_dim(name)
        name, layer = name.split('_')
        transition = self.transitions[int(layer)]
        return transition.get_dim(name)

    @application
    def initial_state(self, state_name, batch_size, *args, **kwargs):
        state_name, layer = state_name.split('_')
        transition = self.transitions[int(layer)]
        return transition.initial_state(state_name, batch_size,
                                        *args, **kwargs)


#-----------------------------------------------------------------------------
# Testing

import numpy
import theano
from theano import tensor
import itertools
from numpy.testing import assert_allclose
from blocks.initialization import Constant
from blocks.bricks.recurrent import LSTM
from collections import OrderedDict

class tRecurrentStack(object):
    def setUp(self):
        depth = 2
        self.depth = depth
        self.sync = False
        dim = 3
        tarnsitions = [LSTM(dim=dim) for _ in range(depth)]
        self.stack = RecurrentStack(tarnsitions,
                                    weights_init=Constant(2),
                                    biases_init=Constant(0))
        self.stack.initialize()

    def test_one_step(self):
        depth = self.depth
        kwargs = OrderedDict()
        kwargs['inputs'] = tensor.matrix('inputs')

        for d in range(depth):
            kwargs['states_' + str(d)] = tensor.matrix('states_' + str(d))
            kwargs['cells_' + str(d)] = tensor.matrix('cells_' + str(d))
        results = self.stack.apply(iterate=False, **kwargs)
        next_h = theano.function(inputs=list(kwargs.values()),
                                 outputs=results)

        # batch=2
        h0_val = 0.1 * numpy.array([[[1, 1, 0], [0, 1, 1]]]*depth,
                                   dtype=theano.config.floatX)
        c0_val = 0.1 * numpy.array([[[1, 1, 0], [0, 1, 1]]]*depth,
                                   dtype=theano.config.floatX)
        x_val = 0.1 * numpy.array([range(12), range(12, 24)],
                                  dtype=theano.config.floatX)
        # we will use same weights on all layers<commit_msg>unique name to each fork layer in RecurrentStack<commit_after>from blocks.bricks.recurrent import BaseRecurrent, recurrent
from blocks.bricks import Initializable, Linear
from blocks.bricks.base import application
from blocks.bricks.parallel import Fork

class RecurrentStack(BaseRecurrent, Initializable):
    u"""Stack of recurrent networks.

    Build a stack of recurrent layers from a supplied list of
    BaseRecurrent objects. Each object must have a `sequences`,
    `contexts`, `states` and `outputs` parameters to its `apply` method.
    It is assumed that the `sequences` of all objects has an "inputs"
    element. It is assumed that all `states` have a "states" element
    (this can be configured with `states_name` parameter.)

    The `sequences` (the "inputs") of the the first layer (layer 0) is
    used as the `sequence` of the stack (the instance of this class.)
    The "mask" element of the `sequences` is carried to all layers that
    have a "mask" element in their `sequences`.
    For all the following layers, all elements of the `sequences`, except
    for "mask", are forked from the "states" element of the of the
    `states` of the previous layer.

    The concatenated `states` of all layers is the `states` of the stack.
    In order to avoid conflict, the names of the elements appearing in the
    `states` of each layers are suffixed with a '_<n>' before combining
    them. The '<n>' is the number of the layer starting from 0.
    The same process is true for the `contexts` and `outputs` of the
    stack.

    Parameters
    ----------
    transitions : list
        List of recurrent units to use in each layer.
        Note: A suffix with layer number is added to transitions' names.
    prototype : :class:`.FeedForward`, optional
        The transformation applied to each states of each layer  when it is
         forked to the sequences of the next layer. By
        default a :class:`.Linear` transformation with biases is used.
    states_name : string
        In a stack of RNN the state of each layer is used as input to the
        next. This string identify which element in the output of each
        layer should be used for this task. By default the element is
        called "states". To be more precise, this is the name of the
        element in the outputs of the apply method of each transition
        (layer) that is used, via fork, as the sequences (input) of the
        next layer. The same element should also appear in the states
        parameter of the apply method.
    sync : bool
        If true, the input of each layer is forked from the states of the
        lower layer from the previous step of the RNN time sequence.
        If false (async, default) the input is forked from the states
        output of the same step.
        Looking at other examples, e.g., it looks like async is the
        correct way.
       https://github.com/karpathy/char-rnn/blob/master/model/LSTM.lua#L25

    Notes
    -----
    See :class:`.BaseRecurrent` for more initialization parameters.

    """
    def __init__(self, transitions, prototype=None, sync=False,
                 states_name="states", **kwargs):
        super(RecurrentStack, self).__init__(**kwargs)

        self.sync = sync
        self.states_name = states_name

        for d, transition in enumerate(transitions):
            transition.name += '_' + str(d)
        self.transitions = transitions

        if prototype is None:
            # By default use Linear (with bias) as a default fork.
            # This overrides the bad default inside Fork which is without bias.
            # Yes I know use_bias=True us the default, but it is a big deal so
            # I write it down explicitly.
            prototype = Linear(use_bias=True)
        depth = len(transitions)
        self.forks = [Fork(self.normal_inputs(d), name='fork_' + str(d),
                           prototype=prototype)
                      for d in range(1, depth)]

        self.children = self.transitions + self.forks

        # Programmatically fix the parameters of the  recurrent decoration
        setattr(self.apply, 'sequences', transitions[0].apply.sequences)
        for t in ["states", "contexts", "outputs"]:
            v = [s + '_' + str(d)
                 for d, transition in enumerate(transitions)
                 for s in getattr(transition.apply, t)]
            setattr(self.apply, t, v)

    def normal_inputs(self, d):
        transition = self.transitions[d]
        return [name for name in transition.apply.sequences
                if 'mask' not in name]

    def _push_allocation_config(self):
        # Configure the forks that connect the "states" element in the `states`
        # of one layer to the elements in the `sequences` of the next layer,
        # excluding "mask".
        # This involves `get_dim` requests
        # to the transitions. To make sure that it answers
        # correctly we should finish its configuration first.
        for transition in self.transitions:
            transition.push_allocation_config()

        for d, fork in enumerate(self.forks):
            fork.input_dim = self.transitions[d].get_dim(self.states_name)
            fork.output_dims = self.transitions[d+1].get_dims(
                fork.output_names)

    @recurrent(sequences=[], states=[], outputs=[], contexts=[])
    def apply(self, **kwargs):
        """Apply the stack of transitions.

        Parameters
        ----------
        All parameters are of type :class:`~tensor.TensorVariable`.

        Returns
        -------
        All return values are of type :class:`~tensor.TensorVariable`.

        """
        results = []
        for d, transition in enumerate(self.transitions):
            sequences_names = transition.apply.sequences
            if d == 0:
                layer_kwargs = dict(
                    (s, kwargs.get(s)) for s in sequences_names)
            else:
                if self.sync:
                    last_states = kwargs[self.states_name + "_" + str(d-1)]
                layer_kwargs = dict(zip(
                    self.normal_inputs(d),
                    self.forks[d-1].apply(last_states, as_list=True)))
                if "mask" in sequences_names:
                    layer_kwargs["mask"] = kwargs.get("mask")

            for t in ["states", "contexts", "outputs"]:
                suffix = '_' + str(d)
                for s in getattr(transition.apply, t):
                    layer_kwargs[s] = kwargs.get(s + suffix)

            result = transition.apply(iterate=False, **layer_kwargs)
            results.extend(result)
            if not self.sync:
                i = transition.apply.outputs.index(self.states_name)
                last_states = result[i]

        return tuple(results)

    def get_dim(self, name):
        if name in self.transitions[0].apply.sequences:
            return self.transitions[0].get_dim(name)
        name, layer = name.split('_')
        transition = self.transitions[int(layer)]
        return transition.get_dim(name)

    @application
    def initial_state(self, state_name, batch_size, *args, **kwargs):
        state_name, layer = state_name.split('_')
        transition = self.transitions[int(layer)]
        return transition.initial_state(state_name, batch_size,
                                        *args, **kwargs)


#-----------------------------------------------------------------------------
# Testing

import numpy
import theano
from theano import tensor
import itertools
from numpy.testing import assert_allclose
from blocks.initialization import Constant
from blocks.bricks.recurrent import LSTM
from collections import OrderedDict

class tRecurrentStack(object):
    def setUp(self):
        depth = 4
        self.depth = depth
        self.sync = False
        dim = 3  # don't change, hardwired in the code
        tarnsitions = [LSTM(dim=dim) for _ in range(depth)]
        self.stack = RecurrentStack(tarnsitions,
                                    weights_init=Constant(2),
                                    biases_init=Constant(0))
        self.stack.initialize()

    def test_one_step(self):
        depth = self.depth
        kwargs = OrderedDict()
        kwargs['inputs'] = tensor.matrix('inputs')

        for d in range(depth):
            kwargs['states_' + str(d)] = tensor.matrix('states_' + str(d))
            kwargs['cells_' + str(d)] = tensor.matrix('cells_' + str(d))
        results = self.stack.apply(iterate=False, **kwargs)
        next_h = theano.function(inputs=list(kwargs.values()),
                                 outputs=results)

        # batch=2
        h0_val = 0.1 * numpy.array([[[1, 1, 0], [0, 1, 1]]]*depth,
                                   dtype=theano.config.floatX)
        c0_val = 0.1 * numpy.array([[[1, 1, 0], [0, 1, 1]]]*depth,
                                   dtype=theano.config.floatX)
        x_val = 0.1 * numpy.array([range(12), range(12, 24)],
                                  dtype=theano.config.floatX)
        # we will use same weights on all layers
{'fraction_non_alphanumeric': 0.09922644554510546, 'fraction_numerical': 0.004786020368412265, 'mean_word_length': 2.116545265348595, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1259385', 'n_tokens_mistral': 4878, 'n_tokens_neox': 4660, 'n_words': 1707}