<commit_before># Library for full image cnn operations

import numpy as np
import mahotas
import time
import h5py

import cPickle
import gzip
import os
import sys
import time

import theano
import theano.tensor as T
from theano.tensor.signal import downsample
from theano.tensor.nnet import conv

class MaxoutMaxpoolLayer(object):
    """Maxout / Maxpool Layer of a convolutional network """

    def __init__(self, input, filter_shape, image_shape, maxoutsize, poolsize, W, b):

        assert image_shape[1] == filter_shape[1]
        self.input = input

        self.W = theano.shared(W, borrow=True)
        self.b = theano.shared(b, borrow=True)

        # Convolve
        conv_out = conv.conv2d(input=input, filters=self.W,
                filter_shape=filter_shape, image_shape=image_shape)

        # Bias (pixel-wise)
        bias_out = conv_out + self.b.dimshuffle('x', 0, 1, 2)

        # Maxout
        maxout_out = None
        for i in xrange(maxoutsize):
            t = bias_out[:,i::maxoutsize,:,:]
            if maxout_out is None:
                maxout_out = t
            else:
                maxout_out = T.maximum(maxout_out, t)

        # Maxpool
        self.output = downsample.max_pool_2d(input=maxout_out, ds=poolsize, ignore_border=True)

        # store parameters of this layer
        self.params = [self.W, self.b]

class SoftmaxLayer(object):
    def __init__(self, input, n_in, n_out, W, b):

        self.input = input

        self.W = theano.shared(W, borrow=True)
        self.b = theano.shared(b, borrow=True)

        lin_output = T.dot(input, self.W) + self.b

        self.output = T.nnet.softmax(lin_output)

        # parameters of the model
        self.params = [self.W, self.b]

class DeepNetwork(object):
    def __init__(self, filename, batch_size=1024):

        network_h5 = h5py.File(filename, 'r')

        self.batch_size = batch_size
        self.nlayers = network_h5['/layers'][...]

        print 'Network has {0} layers.'.format(self.nlayers)

        if '/downsample_factor' in network_h5:
            self.downsample = network_h5['/downsample_factor'][...]
        else:
            self.downsample = 1

        # Calculate network footprint and therefore layer0_size
        footprint = 1
        for layer in range(self.nlayers-1, -1, -1):
            layer_string = '/layer{0}/'.format(layer_i)
            if layer == self.nlayers - 1:
                footprint = network_h5[layer_string + 'ksize'][...][0]
            else:
                footprint = footprint * network_h5[layer_string + 'pool_shape'][...][0] - 1 + network_h5[layer_string + 'kernel_shape'][...][0]<commit_msg>bug fix<commit_after># Library for full image cnn operations

import numpy as np
import mahotas
import time
import h5py

import cPickle
import gzip
import os
import sys
import time

import theano
import theano.tensor as T
from theano.tensor.signal import downsample
from theano.tensor.nnet import conv

class MaxoutMaxpoolLayer(object):
    """Maxout / Maxpool Layer of a convolutional network """

    def __init__(self, input, filter_shape, image_shape, maxoutsize, poolsize, W, b):

        assert image_shape[1] == filter_shape[1]
        self.input = input

        self.W = theano.shared(W, borrow=True)
        self.b = theano.shared(b, borrow=True)

        # Convolve
        conv_out = conv.conv2d(input=input, filters=self.W,
                filter_shape=filter_shape, image_shape=image_shape)

        # Bias (pixel-wise)
        bias_out = conv_out + self.b.dimshuffle('x', 0, 1, 2)

        # Maxout
        maxout_out = None
        for i in xrange(maxoutsize):
            t = bias_out[:,i::maxoutsize,:,:]
            if maxout_out is None:
                maxout_out = t
            else:
                maxout_out = T.maximum(maxout_out, t)

        # Maxpool
        self.output = downsample.max_pool_2d(input=maxout_out, ds=poolsize, ignore_border=True)

        # store parameters of this layer
        self.params = [self.W, self.b]

class SoftmaxLayer(object):
    def __init__(self, input, n_in, n_out, W, b):

        self.input = input

        self.W = theano.shared(W, borrow=True)
        self.b = theano.shared(b, borrow=True)

        lin_output = T.dot(input, self.W) + self.b

        self.output = T.nnet.softmax(lin_output)

        # parameters of the model
        self.params = [self.W, self.b]

class DeepNetwork(object):
    def __init__(self, filename, batch_size=1024):

        network_h5 = h5py.File(filename, 'r')

        self.batch_size = batch_size
        self.nlayers = network_h5['/layers'][...]

        print 'Network has {0} layers.'.format(self.nlayers)

        if '/downsample_factor' in network_h5:
            self.downsample = network_h5['/downsample_factor'][...]
        else:
            self.downsample = 1

        # Calculate network footprint and therefore layer0_size
        footprint = 1
        for layer_i in range(self.nlayers-1, -1, -1):
            layer_string = '/layer{0}/'.format(layer_i)
            if layer_i == self.nlayers - 1:
                footprint = network_h5[layer_string + 'ksize'][...][0]
            else:
                footprint = footprint * network_h5[layer_string + 'pool_shape'][...][0] - 1 + network_h5[layer_string + 'kernel_shape'][...][0]
{'fraction_non_alphanumeric': 0.11220249338874197, 'fraction_numerical': 0.012466943709860219, 'mean_word_length': 2.368320610687023, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4620274', 'n_tokens_mistral': 1759, 'n_tokens_neox': 1680, 'n_words': 408}