<commit_before># -*- coding: UTF-8 -*-

# Copyright (c) 2018 The ungoogled-chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Module for the downloading, checking, and unpacking of necessary files into the source tree
"""

import enum
import urllib.request
import hashlib
from pathlib import Path

from .common import ENCODING, BuildkitError, ExtractorEnum, get_logger
from .extraction import extract_tar_file, extract_with_7z

# Constants


class HashesURLEnum(str, enum.Enum):
    """Enum for supported hash URL schemes"""
    chromium = 'chromium'


# Custom Exceptions


class HashMismatchError(BuildkitError):
    """Exception for computed hashes not matching expected hashes"""
    pass


class _UrlRetrieveReportHook: #pylint: disable=too-few-public-methods
    """Hook for urllib.request.urlretrieve to log progress information to console"""

    def __init__(self):
        self._max_len_printed = 0
        self._last_percentage = None

    def __call__(self, block_count, block_size, total_size):
        downloaded_estimate = block_count * block_size
        percentage = round(downloaded_estimate / total_size, ndigits=3)
        if percentage == self._last_percentage:
            return # Do not needlessly update the console
        self._last_percentage = percentage
        print('\r' + ' ' * self._max_len_printed, end='')
        if total_size > 0:
            status_line = 'Progress: {:.1%} of {:,d} B'.format(percentage, total_size)
        else:
            status_line = 'Progress: {:,d} B of unknown size'.format(downloaded_estimate)
        self._max_len_printed = len(status_line)
        print('\r' + status_line, end='')
<commit_msg>buildkit: Fix pylint 2.2 warnings<commit_after># -*- coding: UTF-8 -*-

# Copyright (c) 2018 The ungoogled-chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Module for the downloading, checking, and unpacking of necessary files into the source tree
"""

import enum
import urllib.request
import hashlib
from pathlib import Path

from .common import ENCODING, BuildkitError, ExtractorEnum, get_logger
from .extraction import extract_tar_file, extract_with_7z

# Constants


class HashesURLEnum(str, enum.Enum):
    """Enum for supported hash URL schemes"""
    chromium = 'chromium'


# Custom Exceptions


class HashMismatchError(BuildkitError):
    """Exception for computed hashes not matching expected hashes"""


class _UrlRetrieveReportHook: #pylint: disable=too-few-public-methods
    """Hook for urllib.request.urlretrieve to log progress information to console"""

    def __init__(self):
        self._max_len_printed = 0
        self._last_percentage = None

    def __call__(self, block_count, block_size, total_size):
        downloaded_estimate = block_count * block_size
        percentage = round(downloaded_estimate / total_size, ndigits=3)
        if percentage == self._last_percentage:
            return # Do not needlessly update the console
        self._last_percentage = percentage
        print('\r' + ' ' * self._max_len_printed, end='')
        if total_size > 0:
            status_line = 'Progress: {:.1%} of {:,d} B'.format(percentage, total_size)
        else:
            status_line = 'Progress: {:,d} B of unknown size'.format(downloaded_estimate)
        self._max_len_printed = len(status_line)
        print('\r' + status_line, end='')

{'fraction_non_alphanumeric': 0.09361702127659574, 'fraction_numerical': 0.00624113475177305, 'mean_word_length': 3.5614489003880982, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6847769', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1022, 'n_words': 325}