<commit_before>import numpy as _np

def ortho_qr(gain=_np.sqrt(2)):
    # gain should be set based on the activation function:
    # linear activations     g = 1 (or greater)
    # tanh activations       g > 1
    # ReLU activations       g = sqrt(2) (or greater)

    def init(shape, **_):
        # Note that this is not strictly correct.
        #
        # What we'd really want is for an initialization which reuses ortho
        # matrices across layers, but we can't have that with the current arch:
        #
        # From <NAME>'s comment in https://plus.google.com/+SoumithChintala/posts/RZfdrRQWL6u
        # > This initialization uses orthogonal matrices, but there’s a bit of
        # > subtlety when it comes to undercomplete layers—basically you need to
        # > make sure that the paths from the input layer to output layer, through
        # > the bottleneck, are preserved. This is accomplished by reusing parts of
        # > the same orthogonal matrices across different layers of the network.
        flat = (shape[0], _np.prod(shape[1:]))
        q1, _ = _np.linalg.qr(_np.random.randn(flat[0], flat[0]))
        q2, _ = _np.linalg.qr(_np.random.randn(flat[1], flat[1]))
        w = _np.dot(q1[:,:min(flat)], q2[:min(flat),:])
        return gain * w.reshape(shape)
    return init

def ortho_svd(gain=_np.sqrt(2)):
    # gain should be set based on the activation function:
    # linear activations     g = 1 (or greater)
    # tanh activations       g > 1
    # ReLU activations       g = sqrt(2) (or greater)

    def init(shape, **_):
        flat = (shape[0], _np.prod(shape[1:]))
        u, _, v = _np.linalg.svd(_np.random.randn(*flat), full_matrices=False)
        w = u if u.shape == flat else v
        return gain * w.reshape(shape)<commit_msg>Fix init argument names for Ortho.<commit_after>import numpy as _np

def ortho_qr(gain=_np.sqrt(2)):
    # gain should be set based on the activation function:
    # linear activations     g = 1 (or greater)
    # tanh activations       g > 1
    # ReLU activations       g = sqrt(2) (or greater)

    def init(shape, fan):
        # Note that this is not strictly correct.
        #
        # What we'd really want is for an initialization which reuses ortho
        # matrices across layers, but we can't have that with the current arch:
        #
        # From <NAME>'s comment in https://plus.google.com/+SoumithChintala/posts/RZfdrRQWL6u
        # > This initialization uses orthogonal matrices, but there’s a bit of
        # > subtlety when it comes to undercomplete layers—basically you need to
        # > make sure that the paths from the input layer to output layer, through
        # > the bottleneck, are preserved. This is accomplished by reusing parts of
        # > the same orthogonal matrices across different layers of the network.
        flat = (shape[0], _np.prod(shape[1:]))
        q1, _ = _np.linalg.qr(_np.random.randn(flat[0], flat[0]))
        q2, _ = _np.linalg.qr(_np.random.randn(flat[1], flat[1]))
        w = _np.dot(q1[:,:min(flat)], q2[:min(flat),:])
        return gain * w.reshape(shape)
    return init

def ortho_svd(gain=_np.sqrt(2)):
    # gain should be set based on the activation function:
    # linear activations     g = 1 (or greater)
    # tanh activations       g > 1
    # ReLU activations       g = sqrt(2) (or greater)

    def init(shape, fan):
        flat = (shape[0], _np.prod(shape[1:]))
        u, _, v = _np.linalg.svd(_np.random.randn(*flat), full_matrices=False)
        w = u if u.shape == flat else v
        return gain * w.reshape(shape)
{'fraction_non_alphanumeric': 0.10850767085076708, 'fraction_numerical': 0.011715481171548118, 'mean_word_length': 2.5681592039800996, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5489958', 'n_tokens_mistral': 1157, 'n_tokens_neox': 1113, 'n_words': 376}