<commit_before># -*- coding: utf-8 -*-
##################################################################
# # Example of usage:
# with Progbar(target=50, name="Prog1").context() as p1:
#     for i in range(200):
#         time.sleep(0.2)
#         p1['value1'] = i
#         p1.add(1)
#         if i == 30:
#             with Progbar(target=20, name="Prog2").context() as p2:
#                 for j in range(60):
#                     time.sleep(0.2)
#                     p2['value2'] = j
#                     p2.add(1)
##################################################################
from __future__ import print_function, division, absolute_import

import sys
import time
from numbers import Number
from datetime import datetime
from contextlib import contextmanager
from collections import OrderedDict, defaultdict

import numpy as np

from odin.visual.bashplot import print_bar

try:
    from tqdm import __version__ as tqdm_version
    tqdm_version = int(tqdm_version.split(".")[0])
    if tqdm_version < 4:
        raise ImportError
    from tqdm import tqdm as _tqdm
    from tqdm._utils import _environ_cols_wrapper
except ImportError:
    sys.stderr.write("[ERROR] Cannot import `tqdm` version >= 4.\n")
    exit()

try:
    import colorama
    colorama.init()
    from colorama import Fore as _Fore
    _RED = _Fore.RED
    _YELLOW = _Fore.YELLOW
    _CYAN = _Fore.CYAN
    _MAGENTA = _Fore.MAGENTA
    _RESET = _Fore.RESET
except ImportError:
    _RED, _YELLOW, _CYAN, _MAGENTA, _RESET = '', '', '', '', ''

_NUMBERS_CH = {
    ord('0'): 0,
    ord('1'): 1,
    ord('2'): 2,
    ord('3'): 3,
    ord('4'): 4,
    ord('5'): 5,
    ord('6'): 6,
    ord('7'): 7,
    ord('8'): 8,
    ord('9'): 9,
}


# ===========================================================================
# Helper
# ===========================================================================
_LAST_UPDATED_PROG = [None]


@contextmanager
def _progbar(pb, print_report, print_summary):
    org_report = pb.print_report
    org_summary = pb.print_summary
    if print_report is not None:
        pb.print_report = print_report
    if print_summary is not None:
        pb.print_summary = print_summary
    yield pb
    pb.pause()
    pb.print_report = org_report
    pb.print_summary = org_summary


def add_notification(msg):
    msg = _CYAN + "\n[%s]Notification:" % \
        datetime.now().strftime('%d/%b-%H:%M:%S') + _RESET + msg + '\n'
    _tqdm.write(msg)


# ===========================================================================
# Progressbar
# ===========================================================================
class Progbar(object):

    """ Original progress bar is just failed illusion of time and
    estimation.

    Parameters
    ----------
    target: int
        total number of steps expected
    interval: float
        Minimum progress display update interval, in seconds.
    keep: bool
        whether to keep the progress bar when the epoch finished
    print_report: bool
        print updated report along with the progress bar for each update
    print_summary: bool
        print epoch summary after each epoch
    name: str or None
        specific name for the progress bar
    """
    FP = sys.stderr

    def __init__(self, target, interval=0.1, keep=False,
                 print_report=False, print_summary=False,
                 name=None):
        self.__pb = None
        self.target = int(target)
        self._seen_so_far = defaultdict(int) # mapping: epoch_idx -> seen_so_far

        n = len(str(self.target))
        self._counter_fmt = '(%%%dd/%%%dd)' % (n, n)

        if name is None:
            name = "Progress-%s" % datetime.utcnow()
        self._name = name

        self.__interval = float(interval)
        self.__keep = keep
        self.print_report = print_report
        self.print_summary = print_summary
        # ====== for history ====== #
        self._report = OrderedDict()
        self._last_report = None
        self._last_print_time = None
        # ====== recording history ====== #
        # dictonary: {epoch_id: {key: [value1, value2, ...]}}
        self._epoch_hist = defaultdict(lambda: defaultdict(list))
        self._epoch_summary = defaultdict(dict)
        self._epoch_idx = 0
        self._epoch_start_time = None

    # ==================== History management ==================== #
    def __getitem__(self, key):
        return self._report.__getitem__(key)

    def __setitem__(self, key, val):
        self._epoch_hist[self.epoch_idx][key].append(val)
        return self._report.__setitem__(key, val)

    def __delitem__(self, key):
        return self._report.__delitem__(key)

    def __iter__(self):
        return self._report.__iter__()

    # ==================== screen control ==================== #
    def interactive(self, print_report=None, print_summary=None):
        return _progbar(self, print_report, print_summary)

    @property
    def epoch_idx(self):
        return self._epoch_idx

    @property
    def nb_epoch(self):
        return self._epoch_idx + 1

    @property
    def name(self):
        return self._name

    @property
    def history(self):
        """ Return history recording all add item (timestamp, key, value)
        to this progress

        Return
        ------
        dictonary: {epoch_id: {key: [value1, value2, ...]}}<commit_msg>Iterable Progbar<commit_after># -*- coding: utf-8 -*-
##################################################################
# # Example of usage:
# with Progbar(target=50, name="Prog1").context() as p1:
#     for i in range(200):
#         time.sleep(0.2)
#         p1['value1'] = i
#         p1.add(1)
#         if i == 30:
#             with Progbar(target=20, name="Prog2").context() as p2:
#                 for j in range(60):
#                     time.sleep(0.2)
#                     p2['value2'] = j
#                     p2.add(1)
##################################################################
from __future__ import print_function, division, absolute_import

import sys
import time
from numbers import Number
from datetime import datetime
from contextlib import contextmanager
from collections import OrderedDict, defaultdict

import numpy as np

from odin.visual.bashplot import print_bar

try:
    from tqdm import __version__ as tqdm_version
    tqdm_version = int(tqdm_version.split(".")[0])
    if tqdm_version < 4:
        raise ImportError
    from tqdm import tqdm as _tqdm
    from tqdm._utils import _environ_cols_wrapper
except ImportError:
    sys.stderr.write("[ERROR] Cannot import `tqdm` version >= 4.\n")
    exit()

try:
    import colorama
    colorama.init()
    from colorama import Fore as _Fore
    _RED = _Fore.RED
    _YELLOW = _Fore.YELLOW
    _CYAN = _Fore.CYAN
    _MAGENTA = _Fore.MAGENTA
    _RESET = _Fore.RESET
except ImportError:
    _RED, _YELLOW, _CYAN, _MAGENTA, _RESET = '', '', '', '', ''

_NUMBERS_CH = {
    ord('0'): 0,
    ord('1'): 1,
    ord('2'): 2,
    ord('3'): 3,
    ord('4'): 4,
    ord('5'): 5,
    ord('6'): 6,
    ord('7'): 7,
    ord('8'): 8,
    ord('9'): 9,
}


# ===========================================================================
# Helper
# ===========================================================================
_LAST_UPDATED_PROG = [None]


@contextmanager
def _progbar(pb, print_report, print_summary):
    org_report = pb.print_report
    org_summary = pb.print_summary
    if print_report is not None:
        pb.print_report = print_report
    if print_summary is not None:
        pb.print_summary = print_summary
    yield pb
    pb.pause()
    pb.print_report = org_report
    pb.print_summary = org_summary


def add_notification(msg):
    msg = _CYAN + "\n[%s]Notification:" % \
        datetime.now().strftime('%d/%b-%H:%M:%S') + _RESET + msg + '\n'
    _tqdm.write(msg)


# ===========================================================================
# Progressbar
# ===========================================================================
class Progbar(object):

    """ Original progress bar is just failed illusion of time and
    estimation.

    Parameters
    ----------
    target: int
        total number of steps expected
    interval: float
        Minimum progress display update interval, in seconds.
    keep: bool
        whether to keep the progress bar when the epoch finished
    print_report: bool
        print updated report along with the progress bar for each update
    print_summary: bool
        print epoch summary after each epoch
    name: str or None
        specific name for the progress bar

    Examples
    --------
    >>> import numpy as np
    >>> from odin.utils import Progbar
    >>> x = list(range(10))
    >>> for i in Progbar(target=x):
    ...     pass
    """
    FP = sys.stderr

    def __init__(self, target, interval=0.1, keep=False,
                 print_report=False, print_summary=False,
                 name=None):
        self.__pb = None # tqdm object
        self.__count_func = lambda x: len(x) # used when target is iterable
        self.__report_func = lambda x: None # used when target is iterable
        if isinstance(target, Number):
            self.target = int(target)
            self.__iter_obj = None
        elif hasattr(target, '__len__'):
            self.target = len(target)
            self.__iter_obj = target
        else:
            raise ValueError("Unsupport for `target` type: %s" %
                             str(target.__class__))

        self._seen_so_far = defaultdict(int) # mapping: epoch_idx -> seen_so_far

        n = len(str(self.target))
        self._counter_fmt = '(%%%dd/%%%dd)' % (n, n)

        if name is None:
            name = "Progress-%s" % datetime.utcnow()
        self._name = name

        self.__interval = float(interval)
        self.__keep = keep
        self.print_report = print_report
        self.print_summary = print_summary
        # ====== for history ====== #
        self._report = OrderedDict()
        self._last_report = None
        self._last_print_time = None
        # ====== recording history ====== #
        # dictonary: {epoch_id: {key: [value1, value2, ...]}}
        self._epoch_hist = defaultdict(lambda: defaultdict(list))
        self._epoch_summary = defaultdict(dict)
        self._epoch_idx = 0
        self._epoch_start_time = None

    # ==================== History management ==================== #
    def __getitem__(self, key):
        return self._report.__getitem__(key)

    def __setitem__(self, key, val):
        self._epoch_hist[self.epoch_idx][key].append(val)
        return self._report.__setitem__(key, val)

    def __delitem__(self, key):
        return self._report.__delitem__(key)

    def set_iter_info(self, count_func=None, report_func=None):
        """
        Parameters
        ----------
        count_func: callable
            a function takes the returned batch and return an integer for upating
            progress.
        report_func: callable
            a function takes the returned batch and a collection of pair
            (key, value) for constructing the report.

        Return
        ------
        Progbar (self) for chaining method
        """
        if count_func is not None:
            if not callable(count_func):
                raise ValueError("`count_func` must be callable or None.")
            self.__count_func = count_func
        if report_func is not None:
            if not callable(report_func):
                raise ValueError("`report_func` must be callable or None.")
            self.__report_func = report_func
        return self

    def __iter__(self):
        if self.__iter_obj is None:
            raise RuntimeError("This Progbar cannot be iterated, "
                               "the set `target` must be iterable.")
        for X in self.__iter_obj:
            count = self.__count_func(X)
            report = self.__report_func(X)
            if report is not None:
                for key, val in report:
                    self[key] = val
            self.add(int(count))
            yield X
        del self.__iter_obj
        del self.__count_func
        del self.__report_func

    # ==================== screen control ==================== #
    def interactive(self, print_report=None, print_summary=None):
        return _progbar(self, print_report, print_summary)

    @property
    def epoch_idx(self):
        return self._epoch_idx

    @property
    def nb_epoch(self):
        return self._epoch_idx + 1

    @property
    def name(self):
        return self._name

    @property
    def history(self):
        """ Return history recording all add item (timestamp, key, value)
        to this progress

        Return
        ------
        dictonary: {epoch_id: {key: [value1, value2, ...]}}
{'fraction_non_alphanumeric': 0.1808743169398907, 'fraction_numerical': 0.00936768149882904, 'mean_word_length': 2.4840902909980964, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7448078', 'n_tokens_mistral': 3769, 'n_tokens_neox': 3630, 'n_words': 994}