<commit_before>import Image
from bitarray import bitarray


class ImageHash(object):
    def __init__(self, path, size=8):
        self.image_path = path
        self.hash_size = size
        self.image = Image.open(path)

    def average_hash(self):
        image = self.image.resize((self.hash_size, self.hash_size), Image.ANTIALIAS).convert("1")
        pixels = list(image.getdata())
        avg = sum(pixels) / len(pixels)

        diff = []
        for pixel in pixels:
            value = 1 if pixel > avg else 0
            diff.append(str(value))

        ba = bitarray("".join(diff), endian='little')
        return ba.tobytes().encode('hex')<commit_msg>Creates distance and is_look_alike methods in imagehash.py<commit_after>from PIL import Image
from bitarray import bitarray


def _hamming_distance(string, other_string):
    return sum(map(lambda x: 0 if x[0] == x[1] else 1, zip(string, other_string)))


class ImageHash(object):
    def __init__(self, path, size=8):
        self.image_path = path
        self.hash_size = size
        self.image = Image.open(path)

    def average_hash(self):
        image = self.image.resize((self.hash_size, self.hash_size), Image.ANTIALIAS).convert("1")
        pixels = list(image.getdata())
        avg = sum(pixels) / len(pixels)

        diff = []
        for pixel in pixels:
            value = 1 if pixel > avg else 0
            diff.append(str(value))

        ba = bitarray("".join(diff), endian='little')
        return ba.tobytes().encode('hex')


def distance(image_path, other_image_path):
    image_hash = ImageHash(image_path).average_hash()
    other_image_hash = ImageHash(other_image_path).average_hash()

    return _hamming_distance(image_hash, other_image_hash)


def is_look_alike(image_path, other_image_path, tolerance=6):
    return distance(image_path, other_image_path) < tolerance
{'fraction_non_alphanumeric': 0.11641630901287553, 'fraction_numerical': 0.006974248927038627, 'mean_word_length': 3.0809628008752736, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3892170', 'n_tokens_mistral': 605, 'n_tokens_neox': 582, 'n_words': 133}