<commit_before>import numpy as np


class Param:

    def __init__(self, a, name=''):
        self.a = a
        self.d = np.zeros_like(a)
        self.m = np.zeros_like(a)
        self.name = name

    # adagrad update
    def step(self, alpha):
        np.clip(self.d, -5, 5, out=self.d)
        self.m += self.d ** 2
        self.a += -alpha * self.d / np.sqrt(self.m + 1e-8)
        self.d = np.zeros_like(self.a)
<commit_msg>RMSProp instead of Adagrad<commit_after>import numpy as np


class Param:

    decay = 0.95

    def __init__(self, a, name=''):
        self.a = a
        self.d = np.zeros_like(a)
        self.m = np.zeros_like(a)
        self.name = name

    def step(self, alpha):
        np.clip(self.d, -5, 5, out=self.d)
        # self.m += self.d ** 2 # adagrad
        self.m = self.decay * self.m + (1 - self.decay) * (self.d ** 2)  # rmsprop
        self.a += -alpha * self.d / np.sqrt(self.m + 1e-8)
        self.d = np.zeros_like(self.a)

{'fraction_non_alphanumeric': 0.15751295336787566, 'fraction_numerical': 0.015544041450777202, 'mean_word_length': 1.8328445747800586, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '597771', 'n_tokens_mistral': 397, 'n_tokens_neox': 383, 'n_words': 86}