<commit_before>from __future__ import print_function,division
import numpy as np
import six.moves.cPickle as pickle
import theano
import theano.tensor as T
from optimize import rmsprop
from ops import tmax,tmin

class Lipshitz_Layer(object):
    def __init__(self, rng, input, n_max, n_in,n_out, W=None, b=None,init=0):
        self.input = input
        if W is None:
            if init == 0:
                min_value = -1.0 / n_in
                max_value = 1.0 / n_in
            elif init == 1:
                min_value = -np.sqrt(3.0/n_in)
                max_value = np.sqrt(3.0/n_in)
            elif init == 2:
                min_value = 0.5 / n_in
                max_value = 1.5 / n_in
            W_values = np.asarray(
                rng.uniform(
                    low=min_value,
                    high=max_value,
                    size=(n_max, n_in, n_out)
                ),
                dtype=theano.config.floatX
            )
            W = theano.shared(value=W_values, name='W', borrow=True)

        if b is None:
            b_values = np.asarray(
                    rng,
                    low=-0.5,
                    high=0.5,
                    size=(n_max,n_out)
                ),
                dtype=theano.config.floatX
            )
            b = theano.shared(value=b_values, name='b', borrow=True)

        self.W = W
        self.b = b
        self.params=[self.W,self.b]
        self.output = (T.dot(self.input, self.W) + self.b).max(axis=1)
        self.pre_gradient_norms=T.sum(abs(self.W),axis=1)
        self.gradient_norms=tmax(self.pre_gradient_norms,1.0)
        self.max_gradient=T.max(self.pre_gradient_norms)
        self.gradient_cost=T.sum(self.gradient_norms-1.0)

class LMLP(object):
    def __init__(self, rng, input, info_layers,params=None,init=0):
        #info_layer has one entry per layer
        #n_max,n_in,n_out<commit_msg>bug in mlp2<commit_after>from __future__ import print_function,division
import numpy as np
import six.moves.cPickle as pickle
import theano
import theano.tensor as T
from optimize import rmsprop
from ops import tmax,tmin

class Lipshitz_Layer(object):
    def __init__(self, rng, input, n_max, n_in,n_out, W=None, b=None,init=0):
        self.input = input
        if W is None:
            if init == 0:
                min_value = -1.0 / n_in
                max_value = 1.0 / n_in
            elif init == 1:
                min_value = -np.sqrt(3.0/n_in)
                max_value = np.sqrt(3.0/n_in)
            elif init == 2:
                min_value = 0.5 / n_in
                max_value = 1.5 / n_in
            W_values = np.asarray(
                rng.uniform(
                    low=min_value,
                    high=max_value,
                    size=(n_max, n_in, n_out)
                ),
                dtype=theano.config.floatX
            )
            W = theano.shared(value=W_values, name='W', borrow=True)

        if b is None:
            b_values = np.asarray(
                rng.uniform(
                    low=-0.5,
                    high=0.5,
                    size=(n_max,n_out)
                ),
                dtype=theano.config.floatX
            )
            b = theano.shared(value=b_values, name='b', borrow=True)

        self.W = W
        self.b = b
        self.params=[self.W,self.b]
        self.output = (T.dot(self.input, self.W) + self.b).max(axis=1)
        self.pre_gradient_norms=T.sum(abs(self.W),axis=1)
        self.gradient_norms=tmax(self.pre_gradient_norms,1.0)
        self.max_gradient=T.max(self.pre_gradient_norms)
        self.gradient_cost=T.sum(self.gradient_norms-1.0)

class LMLP(object):
    def __init__(self, rng, input, info_layers,params=None,init=0):
        #info_layer has one entry per layer
        #n_max,n_in,n_out
{'fraction_non_alphanumeric': 0.10624344176285415, 'fraction_numerical': 0.014428121720881428, 'mean_word_length': 1.5625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2023328', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1268, 'n_words': 245}