<commit_before>from __future__ import division
import sys
import threading
import time
from timeit import default_timer

from ..callbacks import Callback
from ..utils import ignoring


def format_time(t):
    """Format seconds into a human readable form.

    >>> format_time(10.4)
    '10.4s'
    >>> format_time(1000.4)
    '16min 40.4s'
    """
    m, s = divmod(t, 60)
    h, m = divmod(m, 60)
    if h:
        return '{0:2.0f}hr {1:2.0f}min {2:4.1f}s'.format(h, m, s)
    elif m:
        return '{0:2.0f}min {1:4.1f}s'.format(m, s)
    else:
        return '{0:4.1f}s'.format(s)


class ProgressBar(Callback):
    """A progress bar for dask.

    Can be used as a context manager around dask computations.

    Examples
    --------
    >>> with ProgressBar():    # doctest: +SKIP
    ...     out = res.compute()
    [########################################] | 100% Completed | 10.4 s
    """

    def __init__(self, minimum=0, width=40, dt=0.1):
        self._minimum = minimum
        self._width = width
        self._dt = dt

    def _start(self, dsk):
        self._state = None
        self._start_time = default_timer()
        # Start background thread
        self._running = True
        self._timer = threading.Thread(target=self._timer_func)
        self._timer.start()

    def _pretask(self, key, dsk, state):<commit_msg>Improve progressbar docs<commit_after>from __future__ import division
import sys
import threading
import time
from timeit import default_timer

from ..callbacks import Callback
from ..utils import ignoring


def format_time(t):
    """Format seconds into a human readable form.

    >>> format_time(10.4)
    '10.4s'
    >>> format_time(1000.4)
    '16min 40.4s'
    """
    m, s = divmod(t, 60)
    h, m = divmod(m, 60)
    if h:
        return '{0:2.0f}hr {1:2.0f}min {2:4.1f}s'.format(h, m, s)
    elif m:
        return '{0:2.0f}min {1:4.1f}s'.format(m, s)
    else:
        return '{0:4.1f}s'.format(s)


class ProgressBar(Callback):
    """A progress bar for dask.

    Parameters
    ----------
    minimum : int, optional
        Minimum time threshold in seconds before displaying a progress bar.
        Default is 0 (always display)
    width : int, optional
        Width of the bar
    dt : float, optional
        Update resolution in seconds, default is 0.1 seconds

    Examples
    --------

    Below we create a progress bar with a minimum threshold of 1 second before
    displaying. For cheap computations nothing is shown:

    >>> with ProgressBar(minimum=1.0):      # doctest: +SKIP
    ...     out = some_fast_computation.compute()

    But for expensive computations a full progress bar is displayed:

    >>> with ProgressBar(minimum=1.0):      # doctest: +SKIP
    ...     out = some_slow_computation.compute()
    [########################################] | 100% Completed | 10.4 s

    You can also register a progress bar so that it displays for all
    computations:

    >>> pbar = ProgressBar()                # doctest: +SKIP
    >>> pbar.register()                     # doctest: +SKIP
    >>> some_slow_computation.compute()     # doctest: +SKIP
    [########################################] | 100% Completed | 10.4 s
    """

    def __init__(self, minimum=0, width=40, dt=0.1):
        self._minimum = minimum
        self._width = width
        self._dt = dt

    def _start(self, dsk):
        self._state = None
        self._start_time = default_timer()
        # Start background thread
        self._running = True
        self._timer = threading.Thread(target=self._timer_func)
        self._timer.start()

    def _pretask(self, key, dsk, state):
{'fraction_non_alphanumeric': 0.1514984877646412, 'fraction_numerical': 0.03079461094308496, 'mean_word_length': 2.4385633270321363, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4199784', 'n_tokens_mistral': 1219, 'n_tokens_neox': 1125, 'n_words': 333}