<commit_before>import numpy as np

import restrictedBoltzmannMachine as rbm
from batchtrainer import *
from activationfunctions import *
from common import *
from debug import *
from trainingoptions import *

import theano
from theano import tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

theanoFloat  = theano.config.floatX

DEBUG = False

class MiniBatchTrainer(BatchTrainer):

  def __init__(self, input, inputLabels, nrLayers, initialWeights, initialBiases,
               activationFunction, classificationActivationFunction,
               visibleDropout, hiddenDropout,
               adversarial_training, adversarial_epsilon, adversarial_coefficient):
    self.input = input
    self.inputLabels = inputLabels
    # If we should use adversarial training or not
    self.adversarial_training = adversarial_training
    self.adversarial_coefficient = adversarial_coefficient
    self.adversarial_epsilon = adversarial_epsilon

    self.visibleDropout = visibleDropout
    self.hiddenDropout = hiddenDropout
    self.activationFunction = activationFunction
    self.classificationActivationFunction = classificationActivationFunction

    # Let's initialize the fields
    # The weights and biases, make them shared variables
    self.weights = []
    self.biases = []
    nrWeights = nrLayers - 1
    self.nrWeights = nrWeights
    for i in xrange(nrWeights):
      w = theano.shared(value=np.asarray(initialWeights[i],
                                         dtype=theanoFloat),
                        name='W')
      self.weights.append(w)

      b = theano.shared(value=np.asarray(initialBiases[i],
                                         dtype=theanoFloat),
                        name='b')
      self.biases.append(b)

    # Set the parameters of the object
    # Do not set more than this, these will be used for differentiation in the
    # gradient
    self.params = self.weights + self.biases

    # Required for momentum
    # The updates that were performed in the last batch
    # It is important that the order in which we add the oldUpdates is the same
    # as which we add the params
    self.oldUpdates = []
    for i in xrange(nrWeights):
      oldDw = theano.shared(value=np.zeros(shape=initialWeights[i].shape,
                                           dtype=theanoFloat),
                        name='oldDw')
      self.oldUpdates.append(oldDw)

    for i in xrange(nrWeights):
      oldDb = theano.shared(value=np.zeros(shape=initialBiases[i].shape,
                                           dtype=theanoFloat),
                        name='oldDb')
      self.oldUpdates.append(oldDb)

    # Rmsprop
    # The old mean that were performed in the last batch
    self.oldMeanSquares = []
    for i in xrange(nrWeights):
      oldDw = theano.shared(value=np.zeros(shape=initialWeights[i].shape,
                                           dtype=theanoFloat),
                        name='oldDw')
      self.oldMeanSquares.append(oldDw)

    for i in xrange(nrWeights):
      oldDb = theano.shared(value=np.zeros(shape=initialBiases[i].shape,
                                           dtype=theanoFloat),
                        name='oldDb')
      self.oldMeanSquares.append(oldDb)

    # Create a theano random number generator
    # Required to sample units for dropout
    self.theanoRng = RandomStreams(seed=np.random.randint(1, 1000))

    self.output = self.forwardPass(self.input)

    if self.adversarial_training:
      # TODO: since we are using this here maybe we should move
      # this to BatchTrainer
      error = T.sum(self.costFun(self.input, self.inputLabels))
      grad_error = T.grad(error, self.input)
      adversarial_input = self.input + self.adversarial_epsilon * T.sgn(grad_error)
      self.adversarial_output = self.forwardPass(adversarial_input)

  def forwardPass(self, x):
    # Sample from the visible layer
    # Get the mask that is used for the visible units
    if self.visibleDropout in [1.0, 1]:
      currentLayerValues = x
    else:
      dropoutMask = self.theanoRng.binomial(n=1, p=self.visibleDropout,
                                            size=x.shape,
                                            dtype=theanoFloat)
      currentLayerValues = x * dropoutMask

    for stage in xrange(self.nrWeights -1):
      w = self.weights[stage]
      b = self.biases[stage]
      linearSum = T.dot(currentLayerValues, w) + b
      # dropout: give the next layer only some of the units from this layer
      if self.hiddenDropout in  [1.0, 1]:
        currentLayerValues = self.activationFunction.deterministic(linearSum)
      else:
        dropoutMaskHidden = self.theanoRng.binomial(n=1, p=self.hiddenDropout,
                                            size=linearSum.shape,
                                            dtype=theanoFloat)
        currentLayerValues = dropoutMaskHidden * self.activationFunction.deterministic(linearSum)

    # Last layer operations, no dropout in the output
    w = self.weights[self.nrWeights - 1]
    b = self.biases[self.nrWeights - 1]
    linearSum = T.dot(currentLayerValues, w) + b
    currentLayerValues = self.classificationActivationFunction.deterministic(linearSum)

    return currentLayerValues

  def costFun(self, x, y):
    return T.nnet.categorical_crossentropy(x, y)

  # TODO: do I still need to pass the y?
  def cost(self, y):
    output_error = self.costFun(self.output, y)

    if self.adversarial_training:
      adversarial_error = self.costFun(self.adversarial_output, y)
      # return self.adversarial_coefficient * output_error + (1.0 - self.adversarial_coefficient) * adversarial_error
      return adversarial_error
    else:
      return output_error

class ClassifierBatch(object):

  def __init__(self, input, nrLayers, weights, biases,
               visibleDropout, hiddenDropout,
               activationFunction, classificationActivationFunction):

    self.input = input

    self.classificationWeights = classificationWeightsFromTestWeights(weights,
                                            visibleDropout=visibleDropout,
                                            hiddenDropout=hiddenDropout)

    nrWeights = nrLayers - 1

    currentLayerValues = input

    for stage in xrange(nrWeights -1):
      w = self.classificationWeights[stage]
      b = biases[stage]
      linearSum = T.dot(currentLayerValues, w) + b
      currentLayerValues = activationFunction.deterministic(linearSum)

    self.lastHiddenActivations = currentLayerValues
<commit_msg>debugging<commit_after>import numpy as np

import restrictedBoltzmannMachine as rbm
from batchtrainer import *
from activationfunctions import *
from common import *
from debug import *
from trainingoptions import *

import theano
from theano import tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

theanoFloat  = theano.config.floatX

DEBUG = False

class MiniBatchTrainer(BatchTrainer):

  def __init__(self, input, inputLabels, nrLayers, initialWeights, initialBiases,
               activationFunction, classificationActivationFunction,
               visibleDropout, hiddenDropout,
               adversarial_training, adversarial_epsilon, adversarial_coefficient):
    self.input = input
    self.inputLabels = inputLabels
    # If we should use adversarial training or not
    self.adversarial_training = adversarial_training
    self.adversarial_coefficient = adversarial_coefficient
    self.adversarial_epsilon = adversarial_epsilon

    self.visibleDropout = visibleDropout
    self.hiddenDropout = hiddenDropout
    self.activationFunction = activationFunction
    self.classificationActivationFunction = classificationActivationFunction

    # Let's initialize the fields
    # The weights and biases, make them shared variables
    self.weights = []
    self.biases = []
    nrWeights = nrLayers - 1
    self.nrWeights = nrWeights
    for i in xrange(nrWeights):
      w = theano.shared(value=np.asarray(initialWeights[i],
                                         dtype=theanoFloat),
                        name='W')
      self.weights.append(w)

      b = theano.shared(value=np.asarray(initialBiases[i],
                                         dtype=theanoFloat),
                        name='b')
      self.biases.append(b)

    # Set the parameters of the object
    # Do not set more than this, these will be used for differentiation in the
    # gradient
    self.params = self.weights + self.biases

    # Required for momentum
    # The updates that were performed in the last batch
    # It is important that the order in which we add the oldUpdates is the same
    # as which we add the params
    self.oldUpdates = []
    for i in xrange(nrWeights):
      oldDw = theano.shared(value=np.zeros(shape=initialWeights[i].shape,
                                           dtype=theanoFloat),
                        name='oldDw')
      self.oldUpdates.append(oldDw)

    for i in xrange(nrWeights):
      oldDb = theano.shared(value=np.zeros(shape=initialBiases[i].shape,
                                           dtype=theanoFloat),
                        name='oldDb')
      self.oldUpdates.append(oldDb)

    # Rmsprop
    # The old mean that were performed in the last batch
    self.oldMeanSquares = []
    for i in xrange(nrWeights):
      oldDw = theano.shared(value=np.zeros(shape=initialWeights[i].shape,
                                           dtype=theanoFloat),
                        name='oldDw')
      self.oldMeanSquares.append(oldDw)

    for i in xrange(nrWeights):
      oldDb = theano.shared(value=np.zeros(shape=initialBiases[i].shape,
                                           dtype=theanoFloat),
                        name='oldDb')
      self.oldMeanSquares.append(oldDb)

    # Create a theano random number generator
    # Required to sample units for dropout
    self.theanoRng = RandomStreams(seed=np.random.randint(1, 1000))

    self.output = self.forwardPass(self.input)

    if self.adversarial_training:
      # TODO: since we are using this here maybe we should move
      # this to BatchTrainer
      error = T.sum(self.costFun(self.input, self.inputLabels))
      grad_error = T.grad(error, self.input)
      adversarial_input = self.input
      # + self.adversarial_epsilon * T.sgn(grad_error)
      self.adversarial_output = self.forwardPass(adversarial_input)

  def forwardPass(self, x):
    # Sample from the visible layer
    # Get the mask that is used for the visible units
    if self.visibleDropout in [1.0, 1]:
      currentLayerValues = x
    else:
      dropoutMask = self.theanoRng.binomial(n=1, p=self.visibleDropout,
                                            size=x.shape,
                                            dtype=theanoFloat)
      currentLayerValues = x * dropoutMask

    for stage in xrange(self.nrWeights -1):
      w = self.weights[stage]
      b = self.biases[stage]
      linearSum = T.dot(currentLayerValues, w) + b
      # dropout: give the next layer only some of the units from this layer
      if self.hiddenDropout in  [1.0, 1]:
        currentLayerValues = self.activationFunction.deterministic(linearSum)
      else:
        dropoutMaskHidden = self.theanoRng.binomial(n=1, p=self.hiddenDropout,
                                            size=linearSum.shape,
                                            dtype=theanoFloat)
        currentLayerValues = dropoutMaskHidden * self.activationFunction.deterministic(linearSum)

    # Last layer operations, no dropout in the output
    w = self.weights[self.nrWeights - 1]
    b = self.biases[self.nrWeights - 1]
    linearSum = T.dot(currentLayerValues, w) + b
    currentLayerValues = self.classificationActivationFunction.deterministic(linearSum)

    return currentLayerValues

  def costFun(self, x, y):
    return T.nnet.categorical_crossentropy(x, y)

  # TODO: do I still need to pass the y?
  def cost(self, y):
    output_error = self.costFun(self.output, y)

    if self.adversarial_training:
      adversarial_error = self.costFun(self.adversarial_output, y)
      return self.adversarial_coefficient * output_error + (1.0 - self.adversarial_coefficient) * adversarial_error
    else:
      return output_error

class ClassifierBatch(object):

  def __init__(self, input, nrLayers, weights, biases,
               visibleDropout, hiddenDropout,
               activationFunction, classificationActivationFunction):

    self.input = input

    self.classificationWeights = classificationWeightsFromTestWeights(weights,
                                            visibleDropout=visibleDropout,
                                            hiddenDropout=hiddenDropout)

    nrWeights = nrLayers - 1

    currentLayerValues = input

    for stage in xrange(nrWeights -1):
      w = self.classificationWeights[stage]
      b = biases[stage]
      linearSum = T.dot(currentLayerValues, w) + b
      currentLayerValues = activationFunction.deterministic(linearSum)

    self.lastHiddenActivations = currentLayerValues

{'fraction_non_alphanumeric': 0.07439220799141039, 'fraction_numerical': 0.0032211059130301404, 'mean_word_length': 2.4109338216060685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3911385', 'n_tokens_mistral': 3702, 'n_tokens_neox': 3505, 'n_words': 880}