<commit_before>import numpy as np
import scipy.stats
import datetime

from spodernet.interfaces import IAtIterEndObservable, IAtEpochEndObservable, IAtEpochStartObservable
from spodernet.utils.util import Timer
from spodernet.utils.global_config import Config, Backends
from spodernet.frontend import convert_state

from spodernet.utils.logger import Logger
log = Logger('hooks.py.txt')

class AbstractHook(IAtIterEndObservable, IAtEpochEndObservable):
    def __init__(self, name, metric_name, print_every_x_batches):
        self.epoch_errors = []
        self.current_scores = []
        self.name = name
        self.iter_count = 0
        self.print_every = print_every_x_batches
        self.metric_name = metric_name
        self.epoch = 1

        # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
        self.n = 0
        self.epoch_n = 0
        self.mean = 0
        self.M2 = 0
        self.load_backend_specific_functions()

    def load_backend_specific_functions(self):
        if Config.backend == Backends.TORCH:
            from spodernet.backends.torchbackend import convert_state
            self.convert_state = convert_state
        else:
            self.convert_state = lambda x: x

    def calculate_metric(self, state):
        raise NotImplementedError('Classes that inherit from abstract hook need to implement the calcualte metric method.')

    def at_end_of_iter_event(self, state):
        state = self.convert_state(state)
        metric = self.calculate_metric(state)
        #print(metric)

        self.n += 1
        delta = metric - self.mean
        self.mean += delta/self.n
        delta2 = metric - self.mean<commit_msg>Fixed error in torch specific important.<commit_after>import numpy as np
import scipy.stats
import datetime

from spodernet.interfaces import IAtIterEndObservable, IAtEpochEndObservable, IAtEpochStartObservable
from spodernet.utils.util import Timer
from spodernet.utils.global_config import Config, Backends
from spodernet.frontend import convert_state

from spodernet.utils.logger import Logger
log = Logger('hooks.py.txt')

class AbstractHook(IAtIterEndObservable, IAtEpochEndObservable):
    def __init__(self, name, metric_name, print_every_x_batches):
        self.epoch_errors = []
        self.current_scores = []
        self.name = name
        self.iter_count = 0
        self.print_every = print_every_x_batches
        self.metric_name = metric_name
        self.epoch = 1

        # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
        self.n = 0
        self.epoch_n = 0
        self.mean = 0
        self.M2 = 0
        self.load_backend_specific_functions()

    def load_backend_specific_functions(self):
        if Config.backend == Backends.TORCH:
            from spodernet.frontent import convert_state
            self.convert_state = convert_state
        else:
            self.convert_state = lambda x: x

    def calculate_metric(self, state):
        raise NotImplementedError('Classes that inherit from abstract hook need to implement the calcualte metric method.')

    def at_end_of_iter_event(self, state):
        state = self.convert_state(state)
        metric = self.calculate_metric(state)
        #print(metric)

        self.n += 1
        delta = metric - self.mean
        self.mean += delta/self.n
        delta2 = metric - self.mean
{'fraction_non_alphanumeric': 0.07282415630550622, 'fraction_numerical': 0.0053285968028419185, 'mean_word_length': 2.9108796296296298, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4182482', 'n_tokens_mistral': 1017, 'n_tokens_neox': 993, 'n_words': 232}