<commit_before>import itertools
import sys
import time
import numpy as np
import pandas as pd


class OptimisationIterationEvent(object):
    def __init__(self, sequence, trigger="iter"):
        self._seq = sequence
        self._trigger = trigger
        self._next = next(self._seq) if self._seq is not None else np.inf

    def event_handler(self, logger, x):
        raise NotImplementedError

    def __call__(self, logger, x, final=False):
        if ((self._trigger == "iter" and logger._i >= self._next) or
                (self._trigger == "time" and time.time() - logger._start_time >= self._next) or
                final):
            self.event_handler(logger, x)
            self._next = next(self._seq)


class DisplayOptimisation(OptimisationIterationEvent):
    def __init__(self, sequence, trigger="iter"):
        OptimisationIterationEvent.__init__(self, sequence, trigger)
        self._last_disp = (0, time.time())
        print("Iter\tfunc\t\tgrad\t\titer/s\t\tTimestamp")

    def event_handler(self, logger, x):
        # Print and log
        f, g = logger._fg(x)
        iter_per_time = (logger._i - self._last_disp[0]) / (time.time() - self._last_disp[1])
        sys.stdout.write("\r")
        sys.stdout.write(
            "%i\t%e\t%e\t%f\t%s\t" % (logger._i, f, np.linalg.norm(g), iter_per_time, time.ctime()))
        sys.stdout.flush()
        self._last_disp = (logger._i, time.time())

<commit_msg>Iterators now are correct on reloading.<commit_after>import itertools
import sys
import time
import numpy as np
import pandas as pd


class OptimisationIterationEvent(object):
    def __init__(self, sequence, trigger="iter"):
        self._seq = sequence
        self._trigger = trigger
        self._next = next(self._seq) if self._seq is not None else np.inf

    def event_handler(self, logger, x):
        raise NotImplementedError

    def __call__(self, logger, x, final=False):
        if ((self._trigger == "iter" and logger._i >= self._next) or
                (self._trigger == "time" and time.time() - logger._start_time >= self._next) or
                final):
            self.event_handler(logger, x)
            self._next = next(self._seq)
            while self._next < (logger._i if self._trigger == "iter" else time.time() - logger._start_time):
                self._next = next(self._seq)


class DisplayOptimisation(OptimisationIterationEvent):
    def __init__(self, sequence, trigger="iter"):
        OptimisationIterationEvent.__init__(self, sequence, trigger)
        self._last_disp = (0, time.time())
        print("Iter\tfunc\t\tgrad\t\titer/s\t\tTimestamp")

    def event_handler(self, logger, x):
        # Print and log
        f, g = logger._fg(x)
        iter_per_time = (logger._i - self._last_disp[0]) / (time.time() - self._last_disp[1])
        sys.stdout.write("\r")
        sys.stdout.write(
            "%i\t%e\t%e\t%f\t%s\t" % (logger._i, f, np.linalg.norm(g), iter_per_time, time.ctime()))
        sys.stdout.flush()
        self._last_disp = (logger._i, time.time())


{'fraction_non_alphanumeric': 0.1325459317585302, 'fraction_numerical': 0.001968503937007874, 'mean_word_length': 2.621140142517815, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '151836', 'n_tokens_mistral': 970, 'n_tokens_neox': 961, 'n_words': 222}