<commit_before>import warnings
from keras import backend as K
from keras import activations, initializations, regularizers
from keras.layers import Recurrent
from keras.engine import InputSpec
import numpy as np

from constants import S_OP, R2_OP, R3_OP

class TreeCompositionLSTM(Recurrent):
    '''
    Conceptual differences from LSTM:
    1. Tree LSTM does not differentiate between x and h, because tree composition is not applied at every time step (it is applied when the input symbol is a reduce) and when it is applied, there is no "current input".
    2. Input sequences are not the ones being composed, they are operations on the buffer containing elements corresponding to tokens. There isn't one token per timestep like LSTMs.
    3. Single vectors h and c are replaced by a stack and buffer of h and c corresponding to the structure processed so far.
    4. Gates are applied on two or three elements at a time depending on the type of reduce. Accordingly there are two classes of gates: G_2 (two elements) and G_3 (three elements)
    5. G_2 has two forget gates, for each element that can be forgotten and G_3 has three.
    '''
    def __init__(self, stack_limit, buffer_ops_limit, output_dim,
                 init='glorot_uniform', inner_init='orthogonal',
                 forget_bias_init='one', activation='tanh',
                 inner_activation='hard_sigmoid',
                 W_regularizer=None, U_regularizer=None, V_regularizer=None, b_regularizer=None,
                 dropout_W=0., dropout_U=0., dropout_V=0., **kwargs):
        self.stack_limit = stack_limit
        # buffer_ops_limit is the max of buffer_limit and num_ops. This needs to be one value since the initial buffer state and the list of operations need to be concatenated before passing them to TreeCompositionLSTM
        self.buffer_ops_limit = buffer_ops_limit
        self.output_dim = output_dim
        self.init = initializations.get(init)
        self.inner_init = initializations.get(inner_init)
        self.forget_bias_init = initializations.get(forget_bias_init)
        self.activation = activations.get(activation)
        self.inner_activation = activations.get(inner_activation)
        # Make two copies each of W, U and b since regularizers.get() method modifes them!
        W2_regularizer = dict(W_regularizer)
        W3_regularizer = dict(W_regularizer)
        U2_regularizer = dict(U_regularizer)
        U3_regularizer = dict(U_regularizer)
        b2_regularizer = dict(b_regularizer)
        b3_regularizer = dict(b_regularizer)
        # W, U and b get two copies of each corresponding regularizer
        self.W_regularizers = [regularizers.get(W2_regularizer), regularizers.get(W3_regularizer)] if W_regularizer else None
        self.U_regularizers = [regularizers.get(U2_regularizer), regularizers.get(U3_regularizer)] if U_regularizer else None
        self.V_regularizer = regularizers.get(V_regularizer)
        self.b_regularizers = [regularizers.get(b2_regularizer), regularizers.get(b3_regularizer)] if b_regularizer else None
        # TODO: Ensure output_dim = input_dim - 1
        
        self.dropout_W = dropout_W
        self.dropout_U = dropout_U
        self.dropout_V = dropout_V
        if self.dropout_W:
            self.uses_learning_phase = True
        super(TreeCompositionLSTM, self).__init__(**kwargs)
        if self.stateful:
            warnings.warn("Current implementation cannot be stateful. Ignoring stateful=True", RuntimeWarning)
            self.stateful = False
        if self.return_sequences:
            warnings.warn("Current implementation cannot return sequences. Ignoring return_sequences=True", RuntimeWarning)
            self.return_sequences = False

    def get_initial_states(self, x):<commit_msg>Deep copies of weight regularizers<commit_after>import warnings
import copy
from keras import backend as K
from keras import activations, initializations, regularizers
from keras.layers import Recurrent
from keras.engine import InputSpec
import numpy as np

from constants import S_OP, R2_OP, R3_OP

class TreeCompositionLSTM(Recurrent):
    '''
    Conceptual differences from LSTM:
    1. Tree LSTM does not differentiate between x and h, because tree composition is not applied at every time step (it is applied when the input symbol is a reduce) and when it is applied, there is no "current input".
    2. Input sequences are not the ones being composed, they are operations on the buffer containing elements corresponding to tokens. There isn't one token per timestep like LSTMs.
    3. Single vectors h and c are replaced by a stack and buffer of h and c corresponding to the structure processed so far.
    4. Gates are applied on two or three elements at a time depending on the type of reduce. Accordingly there are two classes of gates: G_2 (two elements) and G_3 (three elements)
    5. G_2 has two forget gates, for each element that can be forgotten and G_3 has three.
    '''
    def __init__(self, stack_limit, buffer_ops_limit, output_dim,
                 init='glorot_uniform', inner_init='orthogonal',
                 forget_bias_init='one', activation='tanh',
                 inner_activation='hard_sigmoid',
                 W_regularizer=None, U_regularizer=None, V_regularizer=None, b_regularizer=None,
                 dropout_W=0., dropout_U=0., dropout_V=0., **kwargs):
        self.stack_limit = stack_limit
        # buffer_ops_limit is the max of buffer_limit and num_ops. This needs to be one value since the initial buffer state and the list of operations need to be concatenated before passing them to TreeCompositionLSTM
        self.buffer_ops_limit = buffer_ops_limit
        self.output_dim = output_dim
        self.init = initializations.get(init)
        self.inner_init = initializations.get(inner_init)
        self.forget_bias_init = initializations.get(forget_bias_init)
        self.activation = activations.get(activation)
        self.inner_activation = activations.get(inner_activation)
        # Make two deep copies each of W, U and b since regularizers.get() method modifes them!
        W2_regularizer = copy.deepcopy(W_regularizer)
        W3_regularizer = copy.deepcopy(W_regularizer)
        U2_regularizer = copy.deepcopy(U_regularizer)
        U3_regularizer = copy.deepcopy(U_regularizer)
        b2_regularizer = copy.deepcopy(b_regularizer)
        b3_regularizer = copy.deepcopy(b_regularizer)
        # W, U and b get two copies of each corresponding regularizer
        self.W_regularizers = [regularizers.get(W2_regularizer), regularizers.get(W3_regularizer)] if W_regularizer else None
        self.U_regularizers = [regularizers.get(U2_regularizer), regularizers.get(U3_regularizer)] if U_regularizer else None
        self.V_regularizer = regularizers.get(V_regularizer)
        self.b_regularizers = [regularizers.get(b2_regularizer), regularizers.get(b3_regularizer)] if b_regularizer else None
        # TODO: Ensure output_dim = input_dim - 1
        
        self.dropout_W = dropout_W
        self.dropout_U = dropout_U
        self.dropout_V = dropout_V
        if self.dropout_W:
            self.uses_learning_phase = True
        super(TreeCompositionLSTM, self).__init__(**kwargs)
        if self.stateful:
            warnings.warn("Current implementation cannot be stateful. Ignoring stateful=True", RuntimeWarning)
            self.stateful = False
        if self.return_sequences:
            warnings.warn("Current implementation cannot return sequences. Ignoring return_sequences=True", RuntimeWarning)
            self.return_sequences = False

    def get_initial_states(self, x):
{'fraction_non_alphanumeric': 0.06279434850863422, 'fraction_numerical': 0.00706436420722135, 'mean_word_length': 3.373569794050343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3424334', 'n_tokens_mistral': 2134, 'n_tokens_neox': 2103, 'n_words': 710}