<commit_before>import numpy as np
import theano
import theano.tensor as T
try:
    import cPickle as pickle
except:
    import pickle
import sys

dic = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',\
    'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z',\
    '1','2','3','4','5','6','7','8','9','0','-','.',',','!','?','(',')','\'','"',' ']

text = np.load('hhgttg.npy')
batch = 160

sys.setrecursionlimit(10000)

class lstm(object):
    def __init__(self):
        self.input = T.matrix()
        self.Wi = theano.shared(self.ortho_weight(256)[:72,:])
        self.Wf = theano.shared(self.ortho_weight(256)[:72,:])
        self.Wc = theano.shared(self.ortho_weight(256)[:72,:])
        self.Wo = theano.shared(self.ortho_weight(256)[:72,:])
        self.Ui = theano.shared(self.ortho_weight(256))
        self.Uf = theano.shared(self.ortho_weight(256))
        self.Uc = theano.shared(self.ortho_weight(256))
        self.Uo = theano.shared(self.ortho_weight(256))
        self.bi = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.bf = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.bc = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.bo = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.C0 = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.h0 = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.W2 = theano.shared(self.ortho_weight(256)[:,:72])
        self.b2 = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(72)),dtype=theano.config.floatX))
        self.target = T.matrix()
        
        self.params = [self.Wi,self.Wf,self.Wc,self.Wo,self.Ui,self.Uf,self.Uc,self.Uo,self.bi,self.bf,self.bc,self.bo,self.h0,self.C0,self.W2,self.b2]
        [self.c,self.h_output],_ = theano.scan(fn=self.step,sequences=self.input,outputs_info=[self.C0,self.h0],non_sequences=self.params[:-4])
        self.output =  T.nnet.softmax(T.dot(self.h_output,self.W2)+self.b2)[40:,:]
        self.cost = T.nnet.categorical_crossentropy(self.output,self.target).mean()
        self.updates = self.adam(self.cost,self.params)
        self.train = theano.function([self.input,self.target],self.cost,updates=self.updates,allow_input_downcast=True)
        self.predict = theano.function([self.input],self.output,allow_input_downcast=True)

    def step(self,input,h0,C0,Wi,Wf,Wc,Wo,Ui,Uf,Uc,Uo,bi,bf,bc,bo):
        i = T.nnet.sigmoid(T.dot(input,Wi)+T.dot(h0,Ui)+bi)
        cand = T.tanh(T.dot(input,Wc)+T.dot(h0,Uc)+bc)
        f = T.nnet.sigmoid(T.dot(input,Wf)+T.dot(h0,Uf)+bf)
        c = cand*i+C0*f
        o = T.nnet.sigmoid(T.dot(input,Wo)+T.dot(h0,Uo)+bo)
        h = o*T.tanh(c)
        return c,h

    def ortho_weight(self,ndim):
        bound = np.sqrt(1./ndim)
        W = np.random.randn(ndim, ndim)*bound
        u, s, v = np.linalg.svd(W)
        return u.astype(theano.config.floatX)

    def adam(self, cost, params, lr=0.0002, b1=0.1, b2=0.01, e=1e-8):
        updates = []<commit_msg>fixed order of inputs in theano scan operation<commit_after>import numpy as np
import theano
import theano.tensor as T
try:
    import cPickle as pickle
except:
    import pickle
import sys

dic = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',\
    'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z',\
    '1','2','3','4','5','6','7','8','9','0','-','.',',','!','?','(',')','\'','"',' ']

text = np.load('hhgttg.npy')
batch = 160

sys.setrecursionlimit(10000)

class lstm(object):
    def __init__(self):
        self.input = T.matrix()
        self.Wi = theano.shared(self.ortho_weight(256)[:72,:])
        self.Wf = theano.shared(self.ortho_weight(256)[:72,:])
        self.Wc = theano.shared(self.ortho_weight(256)[:72,:])
        self.Wo = theano.shared(self.ortho_weight(256)[:72,:])
        self.Ui = theano.shared(self.ortho_weight(256))
        self.Uf = theano.shared(self.ortho_weight(256))
        self.Uc = theano.shared(self.ortho_weight(256))
        self.Uo = theano.shared(self.ortho_weight(256))
        self.bi = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.bf = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.bc = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.bo = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.C0 = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.h0 = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(256)),dtype=theano.config.floatX))
        self.W2 = theano.shared(self.ortho_weight(256)[:,:72])
        self.b2 = theano.shared(np.asarray(np.random.uniform(low=-0.1,high=0.1,size=(72)),dtype=theano.config.floatX))
        self.target = T.matrix()
        
        self.params = [self.h0,self.C0,self.Wi,self.Wf,self.Wc,self.Wo,self.Ui,self.Uf,self.Uc,self.Uo,self.bi,self.bf,self.bc,self.bo]
        [self.c,self.h_output],_ = theano.scan(fn=self.step,sequences=self.input,outputs_info=[self.C0,self.h0],non_sequences=self.params[:-4])
        self.output =  T.nnet.softmax(T.dot(self.h_output,self.W2)+self.b2)[40:,:]
        self.cost = T.nnet.categorical_crossentropy(self.output,self.target).mean()
        self.updates = self.adam(self.cost,self.params+[self.W2,self.b2])
        self.train = theano.function([self.input,self.target],self.cost,updates=self.updates,allow_input_downcast=True)
        self.predict = theano.function([self.input],self.output,allow_input_downcast=True)

    def step(self,input,h0,C0,Wi,Wf,Wc,Wo,Ui,Uf,Uc,Uo,bi,bf,bc,bo):
        i = T.nnet.sigmoid(T.dot(input,Wi)+T.dot(h0,Ui)+bi)
        cand = T.tanh(T.dot(input,Wc)+T.dot(h0,Uc)+bc)
        f = T.nnet.sigmoid(T.dot(input,Wf)+T.dot(h0,Uf)+bf)
        c = cand*i+C0*f
        o = T.nnet.sigmoid(T.dot(input,Wo)+T.dot(h0,Uo)+bo)
        h = o*T.tanh(c)
        return c,h

    def ortho_weight(self,ndim):
        bound = np.sqrt(1./ndim)
        W = np.random.randn(ndim, ndim)*bound
        u, s, v = np.linalg.svd(W)
        return u.astype(theano.config.floatX)

    def adam(self, cost, params, lr=0.0002, b1=0.1, b2=0.01, e=1e-8):
        updates = []
{'fraction_non_alphanumeric': 0.23579008576828028, 'fraction_numerical': 0.04070359063817415, 'mean_word_length': 3.9002849002849005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3085595', 'n_tokens_mistral': 3028, 'n_tokens_neox': 2927, 'n_words': 249}