<commit_before>import pickle
import os
import math
import functools
from collections import defaultdict

import numpy as np
import theano
import theano.tensor as T
import matplotlib.pyplot as plt
from tqdm import tqdm
from tabulate import tabulate
from sklearn.externals import joblib


print = functools.partial(print, flush=True)
floatX = theano.config.floatX
one = np.cast[floatX](1)


def free_shared_variable(x):
    x.set_value(np.empty([1 for _ in range(x.ndim - 1)] + [0], dtype=x.dtype))


class RmspropAuto:

    typ = "auto"

    def __init__(self, f, θs, α=0.001, ρ=0.9, ε=1e-8, dec=0.):
        α, ρ, ε, dec = [np.cast[floatX](h) for h in [α, ρ, ε, dec]]

        t = theano.shared(0, name="t")
        t_u = (t, t + 1)

        gs = T.grad(f, θs)

        self.accs = [theano.shared(np.zeros(θ.shape.eval(), dtype=floatX), borrow=True, name="a") for θ in θs]
        acc_us = [(acc, (ρ * acc) + (one - ρ) * T.sqr(g)) for acc, g in zip(self.accs, gs)]

        θ_us = [(θ, θ - ((α / (one + (t_u[1] * dec))) * g / T.sqrt(acc_u[1] + ε))) for θ, g, acc_u in zip(θs, gs, acc_us)]
        self.updates = acc_us + θ_us + [t_u]

    def __call__(self):
        return self.updates

    def free_shared(self):
        for p in self.accs:
            free_shared_variable(p)


class EvepropAuto:

    typ = "auto"

    def __init__(self, f, θs, α=0.001, ρ=0.9, β=0.999, ε=1e-8, dec=0.):
        α, ρ, β, ε, dec = [np.cast[floatX](h) for h in [α, ρ, β, ε, dec]]

        t = theano.shared(0, name="t")
        t_u = (t, t + 1)

        f_prev = theano.shared(np.cast[floatX](0), name="f_prev")

        ch_fact_lbound = T.switch(T.gt(f, f_prev), 1.1, 1/11.)
        ch_fact_ubound = T.switch(T.gt(f, f_prev), 11., 1/1.1)
        f_ch_fact = f / f_prev
        f_ch_fact = T.switch(T.lt(f_ch_fact, ch_fact_lbound), ch_fact_lbound, f_ch_fact)
        f_ch_fact = T.switch(T.gt(f_ch_fact, ch_fact_ubound), ch_fact_ubound, f_ch_fact)
        f_hat = T.switch(T.gt(t_u[1], 1), f_prev * f_ch_fact, f)
        f_u = (f_prev, f_hat)

        d = theano.shared(one, name="d")
        d_den = T.switch(T.gt(f_hat, f_prev), f_prev, f_hat)
        d_t = (β * d) + (one - β) * T.abs_((f_hat - f_prev) / d_den)
        d_t = T.switch(T.gt(t_u[1], 1), d_t, one)
        d_u = (d, d_t)

        gs = T.grad(f, θs)

        self.accs = [theano.shared(np.zeros(θ.shape.eval(), dtype=floatX), borrow=True, name="a") for θ in θs]
        acc_us = [(acc, (ρ * acc) + (one - ρ) * T.sqr(g)) for acc, g in zip(self.accs, gs)]

        θ_us = [(θ, θ - ((α / (one + (t_u[1] * dec))) * g / (T.sqrt(acc_u[1]) * d_t + ε))) for θ, g, acc_u in zip(θs, gs, acc_us)]
        self.updates = acc_us + θ_us + [t_u, f_u, d_u]

    def __call__(self):
        return self.updates

    def free_shared(self):
        for p in self.accs:
            free_shared_variable(p)


class DnaAuto:

    typ = "auto"

    def __init__(self, f, θs, α=0.001, β1=0.9, β2=0.999, β3=0.999, ε=1e-4, dec=0.):
        α, β1, β2, β3, ε, dec = [np.cast[floatX](h) for h in [α, β1, β2, β3, ε, dec]]

        t = theano.shared(0, name="t")
        t_u = (t, t + 1)
<commit_msg>Change default epsilon for dna<commit_after>import pickle
import os
import math
import functools
from collections import defaultdict

import numpy as np
import theano
import theano.tensor as T
import matplotlib.pyplot as plt
from tqdm import tqdm
from tabulate import tabulate
from sklearn.externals import joblib


print = functools.partial(print, flush=True)
floatX = theano.config.floatX
one = np.cast[floatX](1)


def free_shared_variable(x):
    x.set_value(np.empty([1 for _ in range(x.ndim - 1)] + [0], dtype=x.dtype))


class RmspropAuto:

    typ = "auto"

    def __init__(self, f, θs, α=0.001, ρ=0.9, ε=1e-8, dec=0.):
        α, ρ, ε, dec = [np.cast[floatX](h) for h in [α, ρ, ε, dec]]

        t = theano.shared(0, name="t")
        t_u = (t, t + 1)

        gs = T.grad(f, θs)

        self.accs = [theano.shared(np.zeros(θ.shape.eval(), dtype=floatX), borrow=True, name="a") for θ in θs]
        acc_us = [(acc, (ρ * acc) + (one - ρ) * T.sqr(g)) for acc, g in zip(self.accs, gs)]

        θ_us = [(θ, θ - ((α / (one + (t_u[1] * dec))) * g / T.sqrt(acc_u[1] + ε))) for θ, g, acc_u in zip(θs, gs, acc_us)]
        self.updates = acc_us + θ_us + [t_u]

    def __call__(self):
        return self.updates

    def free_shared(self):
        for p in self.accs:
            free_shared_variable(p)


class EvepropAuto:

    typ = "auto"

    def __init__(self, f, θs, α=0.001, ρ=0.9, β=0.999, ε=1e-8, dec=0.):
        α, ρ, β, ε, dec = [np.cast[floatX](h) for h in [α, ρ, β, ε, dec]]

        t = theano.shared(0, name="t")
        t_u = (t, t + 1)

        f_prev = theano.shared(np.cast[floatX](0), name="f_prev")

        ch_fact_lbound = T.switch(T.gt(f, f_prev), 1.1, 1/11.)
        ch_fact_ubound = T.switch(T.gt(f, f_prev), 11., 1/1.1)
        f_ch_fact = f / f_prev
        f_ch_fact = T.switch(T.lt(f_ch_fact, ch_fact_lbound), ch_fact_lbound, f_ch_fact)
        f_ch_fact = T.switch(T.gt(f_ch_fact, ch_fact_ubound), ch_fact_ubound, f_ch_fact)
        f_hat = T.switch(T.gt(t_u[1], 1), f_prev * f_ch_fact, f)
        f_u = (f_prev, f_hat)

        d = theano.shared(one, name="d")
        d_den = T.switch(T.gt(f_hat, f_prev), f_prev, f_hat)
        d_t = (β * d) + (one - β) * T.abs_((f_hat - f_prev) / d_den)
        d_t = T.switch(T.gt(t_u[1], 1), d_t, one)
        d_u = (d, d_t)

        gs = T.grad(f, θs)

        self.accs = [theano.shared(np.zeros(θ.shape.eval(), dtype=floatX), borrow=True, name="a") for θ in θs]
        acc_us = [(acc, (ρ * acc) + (one - ρ) * T.sqr(g)) for acc, g in zip(self.accs, gs)]

        θ_us = [(θ, θ - ((α / (one + (t_u[1] * dec))) * g / (T.sqrt(acc_u[1]) * d_t + ε))) for θ, g, acc_u in zip(θs, gs, acc_us)]
        self.updates = acc_us + θ_us + [t_u, f_u, d_u]

    def __call__(self):
        return self.updates

    def free_shared(self):
        for p in self.accs:
            free_shared_variable(p)


class DnaAuto:

    typ = "auto"

    def __init__(self, f, θs, α=0.001, β1=0.9, β2=0.999, β3=0.999, ε=1e-8, dec=0.):
        α, β1, β2, β3, ε, dec = [np.cast[floatX](h) for h in [α, β1, β2, β3, ε, dec]]

        t = theano.shared(0, name="t")
        t_u = (t, t + 1)

{'fraction_non_alphanumeric': 0.17260668051782005, 'fraction_numerical': 0.024612434073837303, 'mean_word_length': 2.486350974930362, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3784813', 'n_tokens_mistral': 2951, 'n_tokens_neox': 2754, 'n_words': 672}