<commit_before>import theano
import theano.tensor as T
import numpy as np
import random
from inits import *
from dropout import *

floatX = theano.config.floatX
rng = np.random.RandomState()

def initialize_weights(n_in, n_hidden, bias=False, scale=0.01):
    W_xi = normal(n_in, n_hidden, name='W_xi', scale=scale)
    W_hi = orthogonal(n_hidden, n_hidden, name='W_hi')
    W_ci = theano.shared(np.asarray(rng.normal(size=(n_hidden,)) * scale, dtype=floatX), name='W_ci')
    W_xf = normal(n_in, n_hidden, name='W_xf', scale=scale)
    W_hf = orthogonal(n_hidden, n_hidden, name='W_hf')
    W_cf = theano.shared(np.asarray(rng.normal(size=(n_hidden,)) * scale, dtype=floatX), name='W_cf')
    W_xc = normal(n_in, n_hidden, scale=scale, name='W_xc')
    W_hc = orthogonal(n_hidden, n_hidden, name='W_hc')
    W_xo = normal(n_in, n_hidden, scale=scale, name='W_xo')
    W_ho = orthogonal(n_hidden, n_hidden, name='W_ho')
    W_co = theano.shared(np.asarray(rng.normal(size=(n_hidden,)) * scale, dtype=floatX), name='W_co')
    
    if bias:
        b_i = zeros(n_hidden, name='b_i')
        b_f = theano.shared(np.ones((n_hidden,),dtype=floatX) * 1, name='b_f')
        b_c = zeros(n_hidden, name='b_c')
        b_o = zeros(n_hidden, name='b_o')
        return W_xi, W_hi, W_ci, W_xf, W_hf, W_cf, W_xc, W_hc, W_xo, W_ho, W_co, b_i, b_f, b_c, b_o
    else:
        return W_xi, W_hi, W_ci, W_xf, W_hf, W_cf, W_xc, W_hc, W_xo, W_ho, W_co<commit_msg>forget bias 20<commit_after>import theano
import theano.tensor as T
import numpy as np
import random
from inits import *
from dropout import *

floatX = theano.config.floatX
rng = np.random.RandomState()

def initialize_weights(n_in, n_hidden, bias=False, scale=0.01):
    W_xi = normal(n_in, n_hidden, name='W_xi', scale=scale)
    W_hi = orthogonal(n_hidden, n_hidden, name='W_hi')
    W_ci = theano.shared(np.asarray(rng.normal(size=(n_hidden,)) * scale, dtype=floatX), name='W_ci')
    W_xf = normal(n_in, n_hidden, name='W_xf', scale=scale)
    W_hf = orthogonal(n_hidden, n_hidden, name='W_hf')
    W_cf = theano.shared(np.asarray(rng.normal(size=(n_hidden,)) * scale, dtype=floatX), name='W_cf')
    W_xc = normal(n_in, n_hidden, scale=scale, name='W_xc')
    W_hc = orthogonal(n_hidden, n_hidden, name='W_hc')
    W_xo = normal(n_in, n_hidden, scale=scale, name='W_xo')
    W_ho = orthogonal(n_hidden, n_hidden, name='W_ho')
    W_co = theano.shared(np.asarray(rng.normal(size=(n_hidden,)) * scale, dtype=floatX), name='W_co')
    
    if bias:
        b_i = zeros(n_hidden, name='b_i')
        b_f = theano.shared(np.ones((n_hidden,),dtype=floatX) * 20, name='b_f')
        b_c = zeros(n_hidden, name='b_c')
        b_o = zeros(n_hidden, name='b_o')
        return W_xi, W_hi, W_ci, W_xf, W_hf, W_cf, W_xc, W_hc, W_xo, W_ho, W_co, b_i, b_f, b_c, b_o
    else:
        return W_xi, W_hi, W_ci, W_xf, W_hf, W_cf, W_xc, W_hc, W_xo, W_ho, W_co
{'fraction_non_alphanumeric': 0.15342087076710437, 'fraction_numerical': 0.0038009675190048375, 'mean_word_length': 4.371057513914657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2710064', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1237, 'n_words': 245}