<commit_before>#cython: infer_types=True

import sys
from struct import unpack
from itertools import izip
from hashlib import md5


cdef extern from * nogil:
    ctypedef unsigned char uint8_t
    ctypedef unsigned long int uint32_t
    ctypedef unsigned long long int uint64_t


ctypedef uint8_t uint8
ctypedef uint32_t uint32
ctypedef uint64_t uint64


cdef class PHashCombiner(object):
    """Use polynomial hashing to reduce a vector of hashes
    """

    cdef list _coeffs
    cdef _mask

    def __cinit__(self, size, prime=31, bits=64):
        self._coeffs = [prime ** i for i in xrange(size)]
        self._mask = (1 << bits) - 1

    def combine(self, hashes):
        """Combine a list of integer hashes
        """
        ab = sum(h * c for h, c in izip(hashes, self._coeffs))
        return ab & self._mask


cpdef uint64 hash_combine_boost_64(uint64 seed, uint64 v):
    """Hash two 64-bit integers together
    Uses boost::hash_combine algorithm
    """
    return seed ^ (v + 0x9e3779b9ULL + (seed << 6ULL) + (seed >> 2ULL))


cpdef hash_combine_boost(seed, v):
    """Hash two 64-bit integers together
    Uses boost::hash_combine algorithm
    """
    return seed ^ (v + 0x9e3779b9 + (seed << 6) + (seed >> 2))


cpdef uint64 hash_combine_murmur_64(uint64 seed, uint64 v):
    """Hash two 64-bit integers together
    Uses a Murmur-inspired hash function
    """
    cdef uint64 a = (seed ^ v) * 0x9ddfea08eb382d69ULL
    a ^= (a >> 47ULL)
    cdef uint64 b = (v ^ a) * 0x9ddfea08eb382d69ULL
    b ^= (b >> 47ULL)
    b *= 0x9ddfea08eb382d69ULL
    return b


cpdef hash_combine_murmur(seed, v):
    """Hash two 64-bit integers together
    Uses a Murmur-inspired hash function
    """
    a = (seed ^ v) * 0x9ddfea08eb382d69
    a ^= (a >> 47)
    b = (v ^ a) * 0x9ddfea08eb382d69
    b ^= (b >> 47)
    b *= 0x9ddfea08eb382d69
    return b


cpdef hashable(value):
    if not isinstance(value, basestring):
        return repr(value)
    return value


cpdef uint64 hash_md5_64(x, uint64 seed=0):
    """Return value is 128 bits<commit_msg>inline some functions for more speed<commit_after>#cython: infer_types=True

import sys
from struct import unpack
from itertools import izip
from hashlib import md5


cdef extern from * nogil:
    ctypedef unsigned char uint8_t
    ctypedef unsigned long int uint32_t
    ctypedef unsigned long long int uint64_t


ctypedef uint8_t uint8
ctypedef uint32_t uint32
ctypedef uint64_t uint64


cdef class PHashCombiner(object):
    """Use polynomial hashing to reduce a vector of hashes
    """

    cdef list _coeffs
    cdef _mask

    def __cinit__(self, size, prime=31, bits=64):
        self._coeffs = [prime ** i for i in xrange(size)]
        self._mask = (1 << bits) - 1

    def combine(self, hashes):
        """Combine a list of integer hashes
        """
        ab = sum(h * c for h, c in izip(hashes, self._coeffs))
        return ab & self._mask


cpdef inline uint64 hash_combine_boost_64(uint64 seed, uint64 v):
    """Hash two 64-bit integers together
    Uses boost::hash_combine algorithm
    """
    return seed ^ (v + 0x9e3779b9ULL + (seed << 6ULL) + (seed >> 2ULL))


cpdef inline hash_combine_boost(seed, v):
    """Hash two 64-bit integers together
    Uses boost::hash_combine algorithm
    """
    return seed ^ (v + 0x9e3779b9 + (seed << 6) + (seed >> 2))


cpdef inline uint64 hash_combine_murmur_64(uint64 seed, uint64 v):
    """Hash two 64-bit integers together
    Uses a Murmur-inspired hash function
    """
    cdef uint64 a = (seed ^ v) * 0x9ddfea08eb382d69ULL
    a ^= (a >> 47ULL)
    cdef uint64 b = (v ^ a) * 0x9ddfea08eb382d69ULL
    b ^= (b >> 47ULL)
    b *= 0x9ddfea08eb382d69ULL
    return b


cpdef inline hash_combine_murmur(seed, v):
    """Hash two 64-bit integers together
    Uses a Murmur-inspired hash function
    """
    a = (seed ^ v) * 0x9ddfea08eb382d69
    a ^= (a >> 47)
    b = (v ^ a) * 0x9ddfea08eb382d69
    b ^= (b >> 47)
    b *= 0x9ddfea08eb382d69
    return b


cpdef inline hashable(value):
    if not isinstance(value, basestring):
        return repr(value)
    return value


cpdef uint64 hash_md5_64(x, uint64 seed=0):
    """Return value is 128 bits
{'fraction_non_alphanumeric': 0.0975609756097561, 'fraction_numerical': 0.06743185078909612, 'mean_word_length': 3.0028708133971294, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2797094', 'n_tokens_mistral': 1717, 'n_tokens_neox': 1494, 'n_words': 465}