<commit_before>"""Manages progress bars for dvc repo."""
from __future__ import print_function
import logging
from tqdm import tqdm
from copy import deepcopy
from concurrent.futures import ThreadPoolExecutor


class TqdmThreadPoolExecutor(ThreadPoolExecutor):
    """
    Ensure worker progressbars are cleared away properly.
    """

    def __enter__(self):
        """
        Creates a blank initial dummy progress bar if needed so that workers
        are forced to create "nested" bars.
        """
        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
        if blank_bar.pos > 0:
            # already nested - don't need a placeholder bar
            blank_bar.close()
        self.bar = blank_bar
        return super(TqdmThreadPoolExecutor, self).__enter__()

    def __exit__(self, *a, **k):
        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
        self.bar.close()


class Tqdm(tqdm):
    """
    maximum-compatibility tqdm-based progressbars
    """

    def __init__(
        self,
        iterable=None,
        disable=None,
        bytes=False,  # pylint: disable=W0622
        desc_truncate=None,
        leave=None,
        bar_format=(
            "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}"
            " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
        ),
        **kwargs
    ):
        """
        bytes   : shortcut for
            `unit='B', unit_scale=True, unit_divisor=1024, miniters=1`
        desc_truncate  : like `desc` but will truncate to 10 chars
        kwargs  : anything accepted by `tqdm.tqdm()`
        """
        kwargs = deepcopy(kwargs)
        kwargs.setdefault("unit_scale", True)
        if bytes:
            for k, v in dict(
                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
            ).items():
                kwargs.setdefault(k, v)
        if desc_truncate is not None:
            kwargs.setdefault("desc", self.truncate(desc_truncate))
        if disable is None:
            disable = (
                logging.getLogger(__name__).getEffectiveLevel()
                >= logging.CRITICAL
            )
        super(Tqdm, self).__init__(
            iterable=iterable,
            disable=disable,
            leave=leave,
            bar_format=bar_format,<commit_msg>fix unknown total progress<commit_after>"""Manages progress bars for dvc repo."""
from __future__ import print_function
import logging
from tqdm import tqdm
from copy import deepcopy
from concurrent.futures import ThreadPoolExecutor


class TqdmThreadPoolExecutor(ThreadPoolExecutor):
    """
    Ensure worker progressbars are cleared away properly.
    """

    def __enter__(self):
        """
        Creates a blank initial dummy progress bar if needed so that workers
        are forced to create "nested" bars.
        """
        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
        if blank_bar.pos > 0:
            # already nested - don't need a placeholder bar
            blank_bar.close()
        self.bar = blank_bar
        return super(TqdmThreadPoolExecutor, self).__enter__()

    def __exit__(self, *a, **k):
        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
        self.bar.close()


class Tqdm(tqdm):
    """
    maximum-compatibility tqdm-based progressbars
    """
    BAR_FMT_DEFAULT = (
        "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}"
        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
    )
    BAR_FMT_NOTOTAL = "{desc} {bar:b}{n} [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"

    def __init__(
        self,
        iterable=None,
        disable=None,
        bytes=False,  # pylint: disable=W0622
        desc_truncate=None,
        leave=None,
        bar_format=None,
        **kwargs
    ):
        """
        bytes   : shortcut for
            `unit='B', unit_scale=True, unit_divisor=1024, miniters=1`
        desc_truncate  : like `desc` but will truncate to 10 chars
        kwargs  : anything accepted by `tqdm.tqdm()`
        """
        kwargs = deepcopy(kwargs)
        kwargs.setdefault("unit_scale", True)
        if bytes:
            for k, v in dict(
                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
            ).items():
                kwargs.setdefault(k, v)
        if desc_truncate is not None:
            kwargs.setdefault("desc", self.truncate(desc_truncate))
        if disable is None:
            disable = (
                logging.getLogger(__name__).getEffectiveLevel()
                >= logging.CRITICAL
            )
        if bar_format is None:
            if kwargs.get('total', getattr(iterable, '__len__', None)) is None:
                bar_format = self.BAR_FMT_NOTOTAL
            else:
                bar_format = self.BAR_FMT_DEFAULT
        super(Tqdm, self).__init__(
            iterable=iterable,
            disable=disable,
            leave=leave,
            bar_format=bar_format,
{'fraction_non_alphanumeric': 0.1096969696969697, 'fraction_numerical': 0.010505050505050505, 'mean_word_length': 2.161558109833972, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3166877', 'n_tokens_mistral': 1538, 'n_tokens_neox': 1465, 'n_words': 344}