<commit_before>import numpy as np

import theano
import theano.tensor as T
import lasagne

from theano.ifelse import ifelse

from neuralnet import iterate_minibatches

from collections import OrderedDict
import time

EXTRA_INFO = False

class SVRGOptimizer:
    def __init__(self, m, learning_rate, adaptive=True, non_uniform_prob=True):
        self.m = m
        self.learning_rate = learning_rate
        #My guess is adaptive is if we use line search to dynamically decide the learning rate.
        self.adaptive = adaptive
        self.non_uniform_prob = non_uniform_prob
        self.counted_gradient = theano.shared(0)

    def minimize(self, loss, params, X_train, Y_train, X_test, y_test, input_var, target_var, X_val, Y_val, n_epochs=1000, batch_size=100, output_layer=None ):
        self.input_var = input_var
        self.target_var = target_var
        
        num_batches = X_train.shape[0] / batch_size
        n = num_batches

        if EXTRA_INFO:
            print("Learning Rate:{:.2f}".format(self.learning_rate))
            print("Adaptive:{:.2f}".format(self.adaptive))
            print("Non Uniform Prob of mini batch:{:.2f}".format(self.non_uniform_prob))

        self.L = theano.shared(np.cast['float32'](1. / self.learning_rate))
#        self.Ls = [theano.shared(np.cast['float32'](1. / self.learning_rate)) for _  in range(num_batches)]
        self.Ls = [1. / self.learning_rate for _  in range(num_batches)]

        w_updates, mu_updates = self.make_updates(loss, params)

        train_mu = theano.function([self.input_var, self.target_var], loss, updates=mu_updates)
        train_w = theano.function([self.input_var, self.target_var], loss, updates=w_updates)

        prediction = lasagne.layers.get_output(output_layer, deterministic=True)
        acc_fn =      T.mean(T.eq(T.argmax(prediction, axis=1), target_var),dtype=theano.config.floatX)
        train_acc_fn = theano.function([self.input_var, self.target_var], acc_fn)
        val_fn = theano.function([self.input_var, self.target_var], [loss, acc_fn])


        train_error = []
        validation_error = []
        acc_train = []
        acc_val = []
        test_error = []
        acc_test = []
        times = []

        print "NUMBATCHES: ", n

        j = 0

        #what does this do?
        L_fn = self.make_L_fn(loss, params)

        print("Starting training...")
        for epoch in range(n_epochs):

            t = time.time()

            train_err = 0
            train_acc = 0
            train_batches = 0

            if self.non_uniform_prob:
                batches = self.iterate_minibatches(X_train, Y_train, batch_size)
            else:
                batches = iterate_minibatches(X_train, Y_train, batch_size, shuffle=True)

            for batch in batches:
                #every m batches
                if j % self.m == 0:
                    for mu in self.mu:
                        mu.set_value(0 * mu.get_value())

                    for mu_batch in iterate_minibatches(X_train, Y_train, batch_size, shuffle=False):
                        inputs, targets = mu_batch
                        train_mu(inputs, targets)
#                        train_mu(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))
                    
                    for mu in self.mu:
                        mu.set_value(mu.get_value() / n)

                j += 1               
                inputs, targets = batch
                #print "learning_rate: ", 1. / self.L.get_value()

                # what is L?
                L = self.Ls[self.idx]
                self.L.set_value(L)
                
                current_loss, current_acc = val_fn(inputs, targets)
               # current_loss = val_fn(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))

                if self.adaptive: 
                    l_iter = 0
                    while True:

                   # print "learning_rate: ", 1. / self.L.get_value()

                        loss_next, sq_sum = L_fn(inputs, targets)
                       # loss_next, sq_sum = L_fn(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))
                        if loss_next <= current_loss - 0.5 * sq_sum / self.L.get_value():
                            break
                        else:
                            self.L.set_value(self.L.get_value() * 2)

                        l_iter += 1

                if EXTRA_INFO:
                    print("Iterlation of L (learning rate):{:.2f}".format(l_iter))

                # Batch updates for parameters w
                train_err += train_w(inputs, targets)
                # current_err, current_acc = val_fn(inputs, targets)
                train_acc += current_acc

                self.Ls[self.idx] = self.L.get_value()
               # train_err += train_w(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))
                train_batches += 1
            
            val_err = 0
            val_acc = 0
            val_batches = 0
            for i, batch in enumerate(iterate_minibatches(X_val, Y_val, batch_size, shuffle=True)):
                inputs, targets = batch
                current_err, current_acc = val_fn(inputs, targets)
                val_err += current_err
                val_acc += current_acc
               # val_err += val_fn(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))
                val_batches += 1

            test_err = 0
            test_acc = 0
            test_batches = 0
            for i, batch in enumerate(iterate_minibatches(X_test, y_test, batch_size, shuffle=True)):
                inputs, targets = batch
                current_err, current_acc = val_fn(inputs, targets)
                test_err += current_err
                test_acc += current_acc
#                test_err += test_fn(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))
                test_batches += 1

            times.append(time.time() - t)
            print("Epoch {} of {} took {:.3f}s".format(epoch + 1, n_epochs, time.time() - t))

            print("  training loss:\t\t{:.6f}".format(train_err / train_batches))
            print("  train accuracy:\t\t{:.6f}".format(train_acc / train_batches))
<commit_msg>get improve but still crach at batches 67<commit_after>import numpy as np

import theano
import theano.tensor as T
import lasagne

from theano.ifelse import ifelse

from neuralnet import iterate_minibatches

from collections import OrderedDict
import time

EXTRA_INFO =True 

class SVRGOptimizer:
    def __init__(self, m, learning_rate, adaptive=True, non_uniform_prob=True):
        self.m = m
        self.learning_rate = learning_rate
        #My guess is adaptive is if we use line search to dynamically decide the learning rate.
        self.adaptive = adaptive
        self.non_uniform_prob = non_uniform_prob
        self.counted_gradient = theano.shared(0)

    def minimize(self, loss, params, X_train, Y_train, X_test, y_test, input_var, target_var, X_val, Y_val, n_epochs=1000, batch_size=100, output_layer=None ):
        self.input_var = input_var
        self.target_var = target_var
        
        num_batches = X_train.shape[0] / batch_size
        n = num_batches

        if EXTRA_INFO:
            print("Learning Rate:{:.2f}".format(self.learning_rate))
            print("Adaptive:{:.2f}".format(self.adaptive))
            print("Non Uniform Prob of mini batch:{:.2f}".format(self.non_uniform_prob))

        self.L = theano.shared(np.cast['float32'](1. / self.learning_rate))
#        self.Ls = [theano.shared(np.cast['float32'](1. / self.learning_rate)) for _  in range(num_batches)]
        self.Ls = [1. / self.learning_rate for _  in range(num_batches)]

        w_updates, mu_updates = self.make_updates(loss, params)

        train_mu = theano.function([self.input_var, self.target_var], loss, updates=mu_updates)
        train_w = theano.function([self.input_var, self.target_var], loss, updates=w_updates)

        prediction = lasagne.layers.get_output(output_layer, deterministic=True)
        acc_fn =      T.mean(T.eq(T.argmax(prediction, axis=1), target_var),dtype=theano.config.floatX)
        train_acc_fn = theano.function([self.input_var, self.target_var], acc_fn)
        val_fn = theano.function([self.input_var, self.target_var], [loss, acc_fn])


        train_error = []
        validation_error = []
        acc_train = []
        acc_val = []
        test_error = []
        acc_test = []
        times = []

        print "NUMBATCHES: ", n

        j = 0

        #what does this do?
        L_fn = self.make_L_fn(loss, params)

        print("Starting training...")
        for epoch in range(n_epochs):

            t = time.time()

            train_err = 0
            train_acc = 0
            train_batches = 0

            if self.non_uniform_prob:
                batches = self.iterate_minibatches(X_train, Y_train, batch_size)
            else:
                batches = iterate_minibatches(X_train, Y_train, batch_size, shuffle=True)

            for batch in batches:
                #every m batches
                if j % self.m == 0:
                    for mu in self.mu:
                        mu.set_value(0 * mu.get_value())

                    for mu_batch in iterate_minibatches(X_train, Y_train, batch_size, shuffle=False):
                        inputs, targets = mu_batch
                        train_mu(inputs, targets)
#                        train_mu(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))
                    
                    for mu in self.mu:
                        mu.set_value(mu.get_value() / n)

                j += 1               
                inputs, targets = batch
                #print "learning_rate: ", 1. / self.L.get_value()

                # what is L? L is learning rate. Ls is the list storing learning rate.
                L = self.Ls[self.idx]
                self.L.set_value(L)
                
                current_loss, current_acc = val_fn(inputs, targets)

                if self.adaptive: 
                    l_iter = 0
                    while True:
                        loss_next, sq_sum = L_fn(inputs, targets)
                        if loss_next <= current_loss - 0.5 * sq_sum / self.L.get_value():
                            break
                        else:
                            self.L.set_value(self.L.get_value() * 2)

                        l_iter += 1

                if EXTRA_INFO:
		    print "No. of batch:",train_batches
                    print("Iterlation of L (learning rate):{:.2f}".format(l_iter))
                    print "learning_rate: ", 1. / self.L.get_value(),"\n"

                # Batch updates for parameters w
                train_err += train_w(inputs, targets)
                train_acc += current_acc

                self.Ls[self.idx] = self.L.get_value()
                train_batches += 1
            
            val_err = 0
            val_acc = 0
            val_batches = 0
            for i, batch in enumerate(iterate_minibatches(X_val, Y_val, batch_size, shuffle=True)):
                inputs, targets = batch
                current_err, current_acc = val_fn(inputs, targets)
                val_err += current_err
                val_acc += current_acc
               # val_err += val_fn(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))
                val_batches += 1

            test_err = 0
            test_acc = 0
            test_batches = 0
            for i, batch in enumerate(iterate_minibatches(X_test, y_test, batch_size, shuffle=True)):
                inputs, targets = batch
                current_err, current_acc = val_fn(inputs, targets)
                test_err += current_err
                test_acc += current_acc
#                test_err += test_fn(np.array(inputs.todense(), dtype=np.float32), np.array(targets, dtype=np.int32))
                test_batches += 1

            times.append(time.time() - t)
            print("Epoch {} of {} took {:.3f}s".format(epoch + 1, n_epochs, time.time() - t))

            print("  training loss:\t\t{:.6f}".format(train_err / train_batches))
            print("  train accuracy:\t\t{:.6f}".format(train_acc / train_batches))

{'fraction_non_alphanumeric': 0.1034094564168543, 'fraction_numerical': 0.010775168864586683, 'mean_word_length': 1.7804605410239214, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5682501', 'n_tokens_mistral': 3999, 'n_tokens_neox': 3675, 'n_words': 790}