<commit_before>from os import urandom

import struct

from Crypto.Cipher import AES

from block_tools import pkcs7_pad, random_aes_key
from util import chunks

class HashFunction:
    def __init__(self, digest_size, block_size=16):
        self.digest_size = digest_size
        self.block_size = block_size
        self.default_initial_state = urandom(digest_size)
        self.key = random_aes_key()
        self.iv = urandom(16)

    def compress(self, state, block):
        if len(state) != self.digest_size:
            raise ValueError("length of state must equal digest_size")
        if len(block) != self.block_size:
            raise ValueError("length of block must equal block_size")
        cipher = AES.new(self.key, AES.MODE_CBC, self.iv)
        return cipher.encrypt(pkcs7_pad(state + block))[:self.digest_size]

    def produce_padding(self, message_length):
        length_repr = struct.pack(">Q", message_length * 8)
        padding_length = (-message_length % self.block_size) or self.block_size
        padding = bytearray(length_repr.rjust(padding_length, b"\x00"))
        padding[0] |= (1 << 7)
        return bytes(padding[-self.block_size:])

    def __call__(self, message, initial_state=None):
        state = initial_state or self.default_initial_state
        padded_message = message + self.produce_padding(len(message))
        for block in chunks(padded_message, self.block_size):
            state = self.compress(state, block)
        return state<commit_msg>Make hash function deterministic<commit_after>from itertools import cycle, islice

import struct

from Crypto.Cipher import AES

from block_tools import pkcs7_pad
from util import chunks

class HashFunction:
    byte_pattern = bytes.fromhex("0123456789abcdef")

    def __init__(self, digest_size, block_size=16):
        self.digest_size = digest_size
        self.block_size = block_size
        self.default_initial_state = bytes(islice(cycle(self.byte_pattern), digest_size))

    def compress(self, state, block):
        if len(state) != self.digest_size:
            raise ValueError("length of state must equal digest_size")
        if len(block) != self.block_size:
            raise ValueError("length of block must equal block_size")
        key = iv = b"\x00"*16
        cipher = AES.new(key, AES.MODE_CBC, iv)
        return cipher.encrypt(pkcs7_pad(state + block))[:self.digest_size]

    def produce_padding(self, message_length):
        length_repr = struct.pack(">Q", message_length * 8)
        padding_length = (-message_length % self.block_size) or self.block_size
        padding = bytearray(length_repr.rjust(padding_length, b"\x00"))
        padding[0] |= (1 << 7)
        return bytes(padding[-self.block_size:])

    def __call__(self, message, initial_state=None):
        state = initial_state or self.default_initial_state
        padded_message = message + self.produce_padding(len(message))
        for block in chunks(padded_message, self.block_size):
            state = self.compress(state, block)
        return state
{'fraction_non_alphanumeric': 0.09580052493438321, 'fraction_numerical': 0.011811023622047244, 'mean_word_length': 3.276297335203366, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5748464', 'n_tokens_mistral': 946, 'n_tokens_neox': 903, 'n_words': 218}