<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
import json
import hashlib
import six

from scrapy.http import Headers
try:
    from scrapy.utils.python import to_bytes, to_unicode, to_native_str
except ImportError:
    # scrapy < 1.1
    from scrapy.utils.python import unicode_to_str as to_bytes
    from scrapy.utils.python import str_to_unicode as to_unicode

    def to_native_str(text, encoding=None, errors='strict'):
        """ Return str representation of `text`
        (bytes in Python 2.x and unicode in Python 3.x). """
        if six.PY2:
            return to_bytes(text, encoding, errors)
        else:
            return to_unicode(text, encoding, errors)


def dict_hash(obj, start=''):
    """ Return a hash for a dict, based on its contents """
    h = hashlib.sha1(to_bytes(start))
    h.update(to_bytes(obj.__class__.__name__))
    if isinstance(obj, dict):
        for key, value in sorted(obj.items()):
            h.update(to_bytes(key))
            h.update(to_bytes(dict_hash(value)))
    elif isinstance(obj, (list, tuple)):
        for el in obj:
            h.update(to_bytes(dict_hash(el)))
    else:
        # basic types
        if isinstance(obj, bool):
            value = str(int(obj))
        elif isinstance(obj, (six.integer_types, float)):
            value = str(obj)
        elif isinstance(obj, (six.text_type, bytes)):
            value = obj
        else:
            raise ValueError("Unsupported value type: %s" % obj.__class__)
        h.update(to_bytes(value))
    return h.hexdigest()


def _process(value, sha=False):
    if isinstance(value, (six.text_type, bytes)):
        if sha:
            return hashlib.sha1(to_bytes(value)).hexdigest()
        return 'h', hash(value)
    if isinstance(value, dict):
        return {_process(k, sha=True): _process(v, sha) for k, v in value.items()}
    if isinstance(value, (list, tuple, set, frozenset)):
        return [_process(v, sha) for v in value]
    return value


def _fast_hash(value):
    """
    Return a hash for any JSON-serializable value.<commit_msg>sets are not supported<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
import json
import hashlib
import six

from scrapy.http import Headers
try:
    from scrapy.utils.python import to_bytes, to_unicode, to_native_str
except ImportError:
    # scrapy < 1.1
    from scrapy.utils.python import unicode_to_str as to_bytes
    from scrapy.utils.python import str_to_unicode as to_unicode

    def to_native_str(text, encoding=None, errors='strict'):
        """ Return str representation of `text`
        (bytes in Python 2.x and unicode in Python 3.x). """
        if six.PY2:
            return to_bytes(text, encoding, errors)
        else:
            return to_unicode(text, encoding, errors)


def dict_hash(obj, start=''):
    """ Return a hash for a dict, based on its contents """
    h = hashlib.sha1(to_bytes(start))
    h.update(to_bytes(obj.__class__.__name__))
    if isinstance(obj, dict):
        for key, value in sorted(obj.items()):
            h.update(to_bytes(key))
            h.update(to_bytes(dict_hash(value)))
    elif isinstance(obj, (list, tuple)):
        for el in obj:
            h.update(to_bytes(dict_hash(el)))
    else:
        # basic types
        if isinstance(obj, bool):
            value = str(int(obj))
        elif isinstance(obj, (six.integer_types, float)):
            value = str(obj)
        elif isinstance(obj, (six.text_type, bytes)):
            value = obj
        else:
            raise ValueError("Unsupported value type: %s" % obj.__class__)
        h.update(to_bytes(value))
    return h.hexdigest()


def _process(value, sha=False):
    if isinstance(value, (six.text_type, bytes)):
        if sha:
            return hashlib.sha1(to_bytes(value)).hexdigest()
        return 'h', hash(value)
    if isinstance(value, dict):
        return {_process(k, sha=True): _process(v, sha) for k, v in value.items()}
    if isinstance(value, (list, tuple)):
        return [_process(v, sha) for v in value]
    return value


def _fast_hash(value):
    """
    Return a hash for any JSON-serializable value.
{'fraction_non_alphanumeric': 0.11395738568350491, 'fraction_numerical': 0.0038305003591094086, 'mean_word_length': 2.7039007092198584, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4403070', 'n_tokens_mistral': 1293, 'n_tokens_neox': 1266, 'n_words': 370}