<commit_before>
import itertools

import numpy as np
import theano
import theano.tensor as T

from neural.layer.lstm import LSTM
from neural.layer.rnn import RNN

from neural.network._base import BaseAbstraction
from neural.network._optimizer import OptimizerAbstraction


class SutskeverNetwork(OptimizerAbstraction):
    """
    Abstraction for creating recurent neural networks
    """

    def __init__(self, max_output_size=100, **kwargs):
        OptimizerAbstraction.__init__(self, **kwargs)

        self._input = T.tensor3('x')
        self._target = T.imatrix('t')

        # encoder -> decoder
        self._encoder = Encoder(self._input)
        self._decoder = Decoder(self._input, maxlength=max_output_size)

    def test_value(self, x, t):
        self._input.tag.test_value = x
        self._target.tag.test_value = t

    def set_input(self, layer):
        self._encoder.set_input(layer)
        self._decoder.set_input(layer)

    def push_encoder_layer(self, layer):
        self._encoder.push_layer(layer)

    def push_decoder_layer(self, layer):
        self._decoder.push_layer(layer)

    def weight_list(self):
        """
        Create a list containing all the network weights
        """
        return self._encoder.weight_list() + self._decoder.weight_list()

    def forward_pass(self, x):
        (s_enc, b_enc) = self._encoder.forward_pass(x)
        (eois, y) = self._decoder.forward_pass(s_enc, b_enc)

        return (eois, y)

    def _preloss_scanner(self, y_i, eosi_i, t_i, y_pad, dims, time):
        # Get length of y seqence including the first <EOS>
        yend = eosi_i + 1

        # Get length of t seqence including the first <EOS>
        tend = T.argmin(t_i) + 1

        # Create a new y sequence with T elements
        fill_size = T.max([0, tend - yend])
        y2_i = T.zeros((dims, time), dtype='float32')
        # Keep the actual y elements
        y2_i = T.set_subtensor(y2_i[:, :yend], y_i[:, :yend])
        # Add ignore padding to y2 for the remaining elments
        y2_i = T.set_subtensor(y2_i[:, yend:], y_pad)

        # Createa a new t seqnece with T elements
        t2_i = T.zeros((time, ), dtype='int32')
        # Keep the actual t elements
        t2_i = T.set_subtensor(t2_i[:tend], t_i[:tend])
        # Add <EOS> padding to t2 for the remaining elments,
        # the y2 padding will ignore this
        # -- No code, since 0 from T.zeros is <EOS>

        return [y2_i, t2_i]

    def _preloss(self, eosi, y, t):
        # Create an <EOS> collum vector
        y_pad = T.ones((y.shape[1], )) * (0.01 / y.shape[1])
        y_pad = T.set_subtensor(y_pad[0], 0.99)
        y_pad = y_pad.dimshuffle((0, 'x'))

        (y_pad, t_pad), _ = theano.scan(
            fn=self._preloss_scanner,
            sequences=[y, eosi, t],
            outputs_info=[None, None],
            non_sequences=[
                y_pad,
                y.shape[1],<commit_msg>[fix] some float64 errors<commit_after>
import itertools

import numpy as np
import theano
import theano.tensor as T

from neural.layer.lstm import LSTM
from neural.layer.rnn import RNN

from neural.network._base import BaseAbstraction
from neural.network._optimizer import OptimizerAbstraction


class SutskeverNetwork(OptimizerAbstraction):
    """
    Abstraction for creating recurent neural networks
    """

    def __init__(self, max_output_size=100, **kwargs):
        OptimizerAbstraction.__init__(self, **kwargs)

        self._input = T.tensor3('x')
        self._target = T.imatrix('t')

        # encoder -> decoder
        self._encoder = Encoder(self._input)
        self._decoder = Decoder(self._input, maxlength=max_output_size)

    def test_value(self, x, t):
        self._input.tag.test_value = x
        self._target.tag.test_value = t

    def set_input(self, layer):
        self._encoder.set_input(layer)
        self._decoder.set_input(layer)

    def push_encoder_layer(self, layer):
        self._encoder.push_layer(layer)

    def push_decoder_layer(self, layer):
        self._decoder.push_layer(layer)

    def weight_list(self):
        """
        Create a list containing all the network weights
        """
        return self._encoder.weight_list() + self._decoder.weight_list()

    def forward_pass(self, x):
        (s_enc, b_enc) = self._encoder.forward_pass(x)
        (eois, y) = self._decoder.forward_pass(s_enc, b_enc)

        return (eois, y)

    def _preloss_scanner(self, y_i, eosi_i, t_i, y_pad, dims, time):
        # Get length of y seqence including the first <EOS>
        yend = eosi_i + 1

        # Get length of t seqence including the first <EOS>
        tend = T.argmin(t_i) + 1

        # Create a new y sequence with T elements
        y2_i = T.zeros((dims, time), dtype='float32')
        # Keep the actual y elements
        y2_i = T.set_subtensor(y2_i[:, :yend], y_i[:, :yend])
        # Add ignore padding to y2 for the remaining elments
        y2_i = T.set_subtensor(y2_i[:, yend:], y_pad)

        # Createa a new t seqnece with T elements
        t2_i = T.zeros((time, ), dtype='int32')
        # Keep the actual t elements
        t2_i = T.set_subtensor(t2_i[:tend], t_i[:tend])
        # Add <EOS> padding to t2 for the remaining elments,
        # the y2 padding will ignore this
        # -- No code, since 0 from T.zeros is <EOS>

        return [y2_i, t2_i]

    def _preloss(self, eosi, y, t):
        # Create an <EOS> collum vector
        y_pad = T.ones((y.shape[1], ), dtype='float32') * (0.01 / y.shape[1])
        y_pad = T.set_subtensor(y_pad[0], T.constant(0.99, dtype='float32'))
        y_pad = y_pad.dimshuffle((0, 'x'))

        (y_pad, t_pad), _ = theano.scan(
            fn=self._preloss_scanner,
            sequences=[y, eosi, t],
            outputs_info=[None, None],
            non_sequences=[
                y_pad,
                y.shape[1],
{'fraction_non_alphanumeric': 0.11226575809199318, 'fraction_numerical': 0.013117546848381602, 'mean_word_length': 2.4637168141592922, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4909555', 'n_tokens_mistral': 2030, 'n_tokens_neox': 1970, 'n_words': 498}