<commit_before>import base64
import hashlib
import re


__all__ = (
    'DEFAULT_ALGORITHM',
    'RECOGNISED_ALGORITHMS',
    'generate',
    'parse',
)

DEFAULT_ALGORITHM = 'sha384'

RECOGNISED_ALGORITHMS = (
    'sha512',
    'sha384',
    'sha256',
)

_INTEGRITY_PATTERN = re.compile(r'''
    [ \t]*                                  # RFC 5234 (ABNF): WSP
    (?P<algorithm>%(algorithms)s)           # W3C CSP2: hash-algo
    -
    (?P<b64digest>[a-zA-Z0-9+/]+[=]{0,2})   # W3C CSP2: base64-value
    (?P<options>\?[\041-\176]*)?            # RFC 5234 (ABNF): VCHAR
    [ \t]*                                  # RFC 5234 (ABNF): WSP
    ''' % dict(algorithms='|'.join(RECOGNISED_ALGORITHMS)),
    re.VERBOSE,
)


class Hash(object):
    def __new__(cls, algorithm, digest, options=''):
        cls._check_algorithm(algorithm)
        self = object.__new__(cls)
        self._algorithm = algorithm
        self._digest = digest
        self._options = options
        return self

    @staticmethod
    def _check_algorithm(algorithm):
        if algorithm not in RECOGNISED_ALGORITHMS:
            raise ValueError(
                "Unsupported hash algorithm {algorithm!r}, must be one of: {algorithms}"
                .format(
                    algorithm=algorithm,
                    algorithms=", ".join(RECOGNISED_ALGORITHMS),
                ),
            )

    @classmethod
    def fromresource(cls, resource, algorithm=DEFAULT_ALGORITHM, options=''):
        cls._check_algorithm(algorithm)
        hasher = hashlib.new(algorithm, resource)
        digest = hasher.digest()
        return cls(algorithm, digest, options)

    @classmethod
    def fromhash(cls, algorithm, b64digest, options=''):
        digest = base64.standard_b64decode(b64digest)
        return cls(algorithm, digest, options)

    @classmethod
    def fromhashexpr(cls, s):
        m = _INTEGRITY_PATTERN.match(s)
        if not m:
            raise ValueError("Not a valid integrity value: {!r}".format(s))
        algorithm = m.group('algorithm')
        b64digest = m.group('b64digest')
        options = m.group('options') or ''<commit_msg>Improve  checks of digest argument<commit_after>import base64
import hashlib
import re


__all__ = (
    'DEFAULT_ALGORITHM',
    'RECOGNISED_ALGORITHMS',
    'generate',
    'parse',
)

DEFAULT_ALGORITHM = 'sha384'

RECOGNISED_ALGORITHMS = (
    'sha512',
    'sha384',
    'sha256',
)

_INTEGRITY_PATTERN = re.compile(r'''
    [ \t]*                                  # RFC 5234 (ABNF): WSP
    (?P<algorithm>%(algorithms)s)           # W3C CSP2: hash-algo
    -
    (?P<b64digest>[a-zA-Z0-9+/]+[=]{0,2})   # W3C CSP2: base64-value
    (?P<options>\?[\041-\176]*)?            # RFC 5234 (ABNF): VCHAR
    [ \t]*                                  # RFC 5234 (ABNF): WSP
    ''' % dict(algorithms='|'.join(RECOGNISED_ALGORITHMS)),
    re.VERBOSE,
)


class Hash(object):
    def __new__(cls, algorithm, digest, options=''):
        cls._check_algorithm(algorithm)
        cls._check_digest(digest, algorithm)
        self = object.__new__(cls)
        self._algorithm = algorithm
        self._digest = digest
        self._options = options
        return self

    @staticmethod
    def _check_algorithm(algorithm):
        if algorithm not in RECOGNISED_ALGORITHMS:
            raise ValueError(
                "Unsupported hash algorithm {algorithm!r}, must be one of: {algorithms}"
                .format(
                    algorithm=algorithm,
                    algorithms=", ".join(RECOGNISED_ALGORITHMS),
                ),
            )

    @staticmethod
    def _check_digest(digest, algorithm):
        if not isinstance(digest, bytes):
            raise TypeError("Digest must be a binary string")
        h = hashlib.new(algorithm)
        if len(digest) != h.digest_size:
            raise ValueError(
                "Digest length {l1} doesn't match algorithm digest length {l2}"
                .format(
                    l1=len(digest),
                    l2=h.digest_size,
                )
            )

    @classmethod
    def fromresource(cls, resource, algorithm=DEFAULT_ALGORITHM, options=''):
        cls._check_algorithm(algorithm)
        hasher = hashlib.new(algorithm, resource)
        digest = hasher.digest()
        return cls(algorithm, digest, options)

    @classmethod
    def fromhash(cls, algorithm, b64digest, options=''):
        digest = base64.standard_b64decode(b64digest)
        return cls(algorithm, digest, options)

    @classmethod
    def fromhashexpr(cls, s):
        m = _INTEGRITY_PATTERN.match(s)
        if not m:
            raise ValueError("Not a valid integrity value: {!r}".format(s))
        algorithm = m.group('algorithm')
        b64digest = m.group('b64digest')
        options = m.group('options') or ''
{'fraction_non_alphanumeric': 0.11970074812967581, 'fraction_numerical': 0.024106400665004156, 'mean_word_length': 2.1172279792746114, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '482732', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1503, 'n_words': 312}