<commit_before>#!/usr/bin/env python

import lasagne.layers
import theano
import theano.tensor as T

from theano.sandbox.rng_mrg import MRG_RandomStreams
_srng = MRG_RandomStreams(42)

class Dropout(lasagne.layers.Layer):
    """
    Standout version of dropout. Operates in the same way as traditional 
    dropout, but this implementation allows a different dropout probability
    for each unit; and takes as an input the expression for each of these
    dropout probabilities.
    Inputs:
        * incoming - previous layer in the network.
        * incoming_beliefnet - layer encoding dropout probabilities.
        * alpha - hyperparameter controlling scaling
        * beta - hyperparameter controlling bias
        * rescale - after dropping out units, rescale other inputs
    """
    def __init__(self, incoming, incoming_beliefnet, alpha=1.0, beta=0.0, 
            rescale=True, **kwargs):
        lasagne.layers.Layer.__init__(self, incoming, **kwargs)
        self.incoming_beliefnet = incoming_beliefnet
        self.rescale = rescale
        self.alpha = alpha
        self.beta = beta

    def get_output_for(self, input, deterministic=False):
        # get the probabilities from the beliefnet
        self.p = self.alpha*self.incoming_beliefnet
               + self.beta
        # sample a Bernoulli matrix using these probabilities
        if deterministic or self.p == 0:
            # when making predictions, we take the expectation over the 
            # probabilities
            return self.p*input
        else:
            retain_prob = 1. - self.p
            # sample uniform and threshold to sample from many different 
            # probabilities
            self.uniform = _srng.uniform(self.input_shape)
            # keep if less than retain_prob
            self.mask = self.uniform > retain_prob
            # then just apply mask to input
            return input*self.mask

def get_all_beliefnets(output_layer, input_var):
    """
    Takes an output layer and collects up all the belief net layers in the 
    network. Useful for gathering up the parameters so that they can then 
    be updated.
    """
    all_layers = lasagne.layers.get_all_layers(output_layer, input_var)
    all_beliefnets = [l.incoming_beliefnet for l in all_layers 
                      if isinstance(l, Dropout)]
    return all_beliefnets

def get_all_parameters(output_layer):
    """
    Analogous to the lasagne.layers.get_all_parameters, but gathers only the 
    parameters of the standout layers.
    """
    all_beliefnets = get_all_beliefnets(output_layer)
    return [l.W for l in all_beliefnets]<commit_msg>algorithm 1 layer is not pretty<commit_after>#!/usr/bin/env python

import lasagne.layers
import theano
import theano.tensor as T

from theano.sandbox.rng_mrg import MRG_RandomStreams
_srng = MRG_RandomStreams(42)

class Dropout(lasagne.layers.Layer):
    """
    Standout version of dropout. Operates in the same way as traditional 
    dropout, but this implementation allows a different dropout probability
    for each unit; and takes as an input the expression for each of these
    dropout probabilities.
    Inputs:
        * incoming - previous layer in the network.
        * incoming_beliefnet - layer encoding dropout probabilities.
        * p - probabilities (output of the belief net)
    """
    def __init__(self, incoming, p, **kwargs):
        lasagne.layers.Layer.__init__(self, incoming, **kwargs)
        # don't like passing both of these, but need to monkey patch the mask
        # later
        self.p = p
        self.mask = sample_mask(self.p)

    def patch_mask(self, mask):
        """
        Patch the mask with whatever you want to put in there. It's like monkey
        patching, but it looks totally legit.
        """
        self.mask = mask 
        return mask

    def get_output_for(self, input, deterministic=False):
        if deterministic or T.mean(self.p) == 0:
            # when making predictions, we take the expectation over the 
            # probabilities
            return self.p*input
        elif self.mask.ndim > 2:
            masked = input.dimshuffle(0,1,'x')*self.mask
            return masked.dimshuffle(2,0,1)
        else:
            return input*self.mask

def DropoutAlgorithm2(lasagne.layers.Layer):
    """
    Algorithm 2 reuses parameters from the previous hidden layer to perform
    the forward prop used for dropout probabilities. Then, these are scaled 
    with alpha and beta 
    Inputs:
        * incoming - previous layer
    """
    def __init__(self, incoming, alpha, beta, nonlinearity=lasagne.nonlinearities.sigmoid, **kwargs):
        lasagne.layers.Layer.__init__(self, incoming, **kwargs)
        self.W = incoming.W
        self.alpha = alpha
        self.beta = beta
        self.pi = self.alpha*self.W + self.beta
        self.nonlinearity = nonlinearity

    def get_output_for(self, input, deterministic=False):
        self.p = T.nonlinearity(T.dot(input, self.pi))
        self.mask = sample_mask(self.p)
        if deterministic or T.mean(self.p) == 0:
            return self.p*input
        else:
            return input*self.mask

def sample_mask(p):
    """
    give a matrix of probabilities, this will sample a mask. Theano.
    """
    retain_prob = 1. - p
    # sample uniform and threshold to sample from many different 
    # probabilities
    uniform = _srng.uniform(p.shape)
    # keep if less than retain_prob
    mask = uniform > retain_prob
    return mask

def get_all_beliefnets(output_layer, input_var):
    """
    Takes an output layer and collects up all the belief net layers in the 
    network. Useful for gathering up the parameters so that they can then 
    be updated.
    """
    all_layers = lasagne.layers.get_all_layers(output_layer, input_var)
    all_beliefnets = [l.incoming_beliefnet for l in all_layers 
                      if isinstance(l, Dropout)]
    return all_beliefnets

def get_all_parameters(output_layer):
    """
    Analogous to the lasagne.layers.get_all_parameters, but gathers only the 
    parameters of the standout layers.
    """
    all_beliefnets = get_all_beliefnets(output_layer)
    return [l.W for l in all_beliefnets]

def get_mask_updates(output_layer):
    """
    We have to be careful about mask updates, the mask is only sampled _once_
    per minitbatch, despite running N forward props; for N hidden units.
    """
{'fraction_non_alphanumeric': 0.07163993147484815, 'fraction_numerical': 0.0034262575922753466, 'mean_word_length': 2.607865168539326, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3469293', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1756, 'n_words': 604}