<commit_before>import numpy as np
from deepx import T

def initialize_weights(name, shape, value=None, **kwargs):
    return globals()[name](shape, **kwargs)

def get_fans(shape, dim_ordering='th'):
    if len(shape) == 2:
        fan_in = shape[0]
        fan_out = shape[1]
    elif len(shape) == 4 or len(shape) == 5:
        # Assuming convolution kernels (2D or 3D).
        # TH kernel shape: (depth, input_depth, ...)
        # TF kernel shape: (..., input_depth, depth)
        if dim_ordering == 'th':
            receptive_field_size = np.prod(shape[2:])
            fan_in = shape[1] * receptive_field_size
            fan_out = shape[0] * receptive_field_size
        elif dim_ordering == 'tf':
            receptive_field_size = np.prod(shape[:2])
            fan_in = shape[-2] * receptive_field_size
            fan_out = shape[-1] * receptive_field_size
        else:
            raise ValueError('Invalid dim_ordering: ' + dim_ordering)
    else:
        # No specific assumptions.
        fan_in = np.sqrt(np.prod(shape))
        fan_out = np.sqrt(np.prod(shape))
    return fan_in, fan_out

def uniform(shape, scale=0.05):
    return T.random_uniform(shape, minval=-scale, maxval=scale)


def normal(shape, scale=0.05):
    return T.random_normal(shape, mean=0.0, cov=scale)


def lecun_uniform(shape):
    fan_in, fan_out = get_fans(shape)
    scale = np.sqrt(3. / fan_in)
    return uniform(shape, scale)


def glorot_normal(shape):
    fan_in, fan_out = get_fans(shape)
    s = np.sqrt(0.1 / (fan_in + fan_out))
    return T.random_normal(shape, s)


def glorot_uniform(shape):
    fan_in, fan_out = get_fans(shape)
    s = np.sqrt(6. / (fan_in + fan_out))
    return uniform(shape, s)


def he_normal(shape):<commit_msg>added xavier init<commit_after>import numpy as np
from deepx import T

def initialize_weights(name, shape, value=None, **kwargs):
    return globals()[name](shape, **kwargs)

def get_fans(shape, dim_ordering='th'):
    if len(shape) == 2:
        fan_in = shape[0]
        fan_out = shape[1]
    elif len(shape) == 4 or len(shape) == 5:
        # Assuming convolution kernels (2D or 3D).
        # TH kernel shape: (depth, input_depth, ...)
        # TF kernel shape: (..., input_depth, depth)
        if dim_ordering == 'th':
            receptive_field_size = np.prod(shape[2:])
            fan_in = shape[1] * receptive_field_size
            fan_out = shape[0] * receptive_field_size
        elif dim_ordering == 'tf':
            receptive_field_size = np.prod(shape[:2])
            fan_in = shape[-2] * receptive_field_size
            fan_out = shape[-1] * receptive_field_size
        else:
            raise ValueError('Invalid dim_ordering: ' + dim_ordering)
    else:
        # No specific assumptions.
        fan_in = np.sqrt(np.prod(shape))
        fan_out = np.sqrt(np.prod(shape))
    return fan_in, fan_out

def uniform(shape, scale=0.05):
    return T.random_uniform(shape, minval=-scale, maxval=scale)


def normal(shape, scale=0.05):
    return T.random_normal(shape, mean=0.0, cov=scale)


def lecun_uniform(shape):
    fan_in, fan_out = get_fans(shape)
    scale = np.sqrt(3. / fan_in)
    return uniform(shape, scale)

def xavier(shape, constant=1):
    """ Xavier initialization of network weights"""
    fan_in, fan_out = shape
    low = -constant*np.sqrt(6.0/(fan_in + fan_out))
    high = constant*np.sqrt(6.0/(fan_in + fan_out))
    return T.random_uniform((fan_in, fan_out),
                             minval=low, maxval=high,
                             dtype=T.floatx())

def glorot_normal(shape):
    fan_in, fan_out = get_fans(shape)
    s = np.sqrt(0.1 / (fan_in + fan_out))
    return T.random_normal(shape, s)


def glorot_uniform(shape):
    fan_in, fan_out = get_fans(shape)
    s = np.sqrt(6. / (fan_in + fan_out))
    return uniform(shape, s)


def he_normal(shape):
{'fraction_non_alphanumeric': 0.12847132104853362, 'fraction_numerical': 0.014274591227614845, 'mean_word_length': 2.7453838678328473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6014090', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1309, 'n_words': 298}