<commit_before># -*- coding: utf-8 -*-
"""
imagedict
~~~~~~~~~

Emulates the basic behavior of a dict but with images as keys.

:copyright: (c) 2013 by <NAME>.
:license: MIT, see LICENSE for more details.
"""
from collections import namedtuple

import cv2
import numpy as np


class ImageDict(object):
    _KeyPackage = namedtuple("_KeyPackage", ('image', 'mask', 'fingerprint', 'value'))
    _Fingerprint = namedtuple("_Fingerprint", ('keypoints', 'descriptors'))

    def __init__(self):
        self._keypackages = list()
        self._detector = cv2.FastFeatureDetector()
        self._extractor = cv2.DescriptorExtractor_create('BRISK')
        self._matcher = cv2.BFMatcher(cv2.NORM_L2SQR)

    def __len__(self):
        return len(self._keypackages)

    def __setitem__(self, key, value):
        """Set the given key and value.

        Allowed formats for the key are specified in the parse method
        Allowed formats for image and mask are specified in validate method

        Warning: providing a mask of None is valid, so make sure you actually
        have an image in the mask if you intend to use one.

        Note: a lot of validation is done to avoid confusing errors arising later

        """
        image, mask = self._parse_key_arg(key)
        self._validate_image_and_mask(image, mask)
        #if everything is ok, then create and store the package
        fingerprint = self._fingerprint(image, mask)
        new_keypackage = self._KeyPackage(image, mask, fingerprint, value)
        #overwrite the key if it exists
        for i, existing_kp in enumerate(self._keypackages):
            same_image = np.all(image == existing_kp.image)
            same_mask = np.all(mask == existing_kp.mask)
            if  same_image and same_mask:
                self._keypackages[i] = new_keypackage
                return #finished if a key is overwritten
        self._keypackages.append(new_keypackage)

    def get(self, key, d = None):
        """Get the value for the given key or provide the default value."""
        try:
            return self.__getitem__(key)
        except KeyError:
            return d

    def setdefault(self, key, d):
        """D.setdefault(k,d) -> D.get(k,d), also set D[k]=d if k not in D

        Due to the keys being images, the default is required as there is no
        standard meaning for a default image.
        """
        try:
            value = self.__getitem__(key)
        except KeyError:
            self.__setitem__(key, d)
            value = d
        return value


    def __getitem__(self, key):
        """Get the value for the given key or raise KeyError if not found.

        Allowed formats for the key are specified in the parse method
        Allowed formats for image and mask are specified in validate method

        Warning: providing a mask of None is valid, so make sure you actually
        have an image in the mask if you intend to use one.

        Note: a lot of validation is done to avoid confusing errors arising later

        """
        MINIMUM_GOOD_MATCHES = 3
        MINIMUM_MATCH_QUALITY = 0.1

        scene_image, scene_mask = self._parse_key_arg(key)
        self._validate_image_and_mask(scene_image, scene_mask)
        scene_keypoints, scene_descriptors = self._fingerprint(scene_image, scene_mask)
        if not self._keypackages:
            raise KeyError('No keys in this image dictionary.')
        #precalculate some scene properties
        scene_w, scene_h = scene_image.shape[0:2]
        scene_corners = np.float32([(0,0), (scene_w, 0), (scene_w, scene_h), (0, scene_h)])
        #for each key, decide if it has a good match in the scene
        matching_keypackages_with_quality = list()
        for kp in self._keypackages:
            obj_keypoints, obj_descriptors = kp.fingerprint
            matches = self._matcher.match(obj_descriptors, scene_descriptors)
            if len(matches) < MINIMUM_GOOD_MATCHES:
                continue
            #do some filtering of the matches to find the best ones
            distances = [match.distance for match in matches]
            min_dist = min(distances)
            avg_dist = sum(distances) / len(distances)
            min_dist = min_dist or avg_dist * 0.01 #if min_dist is zero, use a small percentage of avg instead
            good_matches = [match for match in matches if match.distance <= 3 * min_dist]
            #stop this keypackage if there are not enough matched points
            if len(good_matches) < MINIMUM_GOOD_MATCHES:
                continue
            #get the corresponding match positions
            obj_matched_points = np.array([obj_keypoints[match.queryIdx].pt for match in good_matches])
            scene_matched_points = np.array([scene_keypoints[match.trainIdx].pt for match in good_matches])
            #make a homography that describes how the key is oriented in the scene (rotated, scaled, etc.)
            homography, homography_inliers_mask = cv2.findHomography(obj_matched_points, scene_matched_points, cv2.RANSAC, 2.0) #2.0: should be very close
            inliers, total_good_matches = np.sum(homography_inliers_mask), len(homography_inliers_mask)
            #find the actual shape of the found key in the scene
            obj_w, obj_h = kp.image.shape[0:2]
            obj_corners = np.float32([(0,0), (obj_w, 0), (obj_w, obj_h), (0, obj_h)])
            found_obj_corners = cv2.perspectiveTransform(obj_corners.reshape(1, -1, 2), homography).reshape(-1, 2)
            #calculate the quality of the image match and keep/reject this keypackage
            quality = self._match_quality(obj_corners, found_obj_corners, inliers, total_good_matches)
            if quality < MINIMUM_MATCH_QUALITY:
                continue #finished with this key
            #store this keypackage and the quality of the match
            matching_keypackages_with_quality.append((kp, quality))
        #choose the best match from all qualifying matches
        try:
            best_kp, best_quality = max(matching_keypackages_with_quality, key=lambda x: x[1])
        except ValueError:
            raise KeyError('No good match found.')
        return best_kp.value

    def __delitem__(self, key):
        """Delete the provided key. Raise KeyError if it doesn't exist."""
        image, mask = self._parse_key_arg(key)
        self._validate_image_and_mask(image, mask)
        #find and delete the key
        for i, existing_kp in enumerate(self._keypackages):
            same_image = np.all(image == existing_kp.image)
            same_mask = np.all(mask == existing_kp.mask)
            if  same_image and same_mask:
                del(self._keypackages[i])
                return
        raise KeyError('The provided key was not found.')

    def __iter__(self):
        """Provide an iterator over the keys."""
        for kp in self._keypackages:
            yield (kp.image, kp.mask)

    def iterkeys(self):
        return self.__iter__()

    def keys(self):
        return list(self.__iter__())

    def itervalues(self):
        for kp in self._keypackages:
            yield kp.value

    def values(self):
        return list(self.itervalues())

    def iteritems(self):
        for kp in self._keypackages:
            yield ((kp.image, kp.mask), kp.value)

    def items(self):
        return list(self.iteritems())

    def __contains__(self, key):
        """Return True if the image, mask combination is present."""
        image, mask = self._parse_key_arg(key)
        self._validate_image_and_mask(image, mask)
        for kp in self._keypackages:
            same_image = np.all(image == kp.image)
            same_mask = np.all(mask == kp.mask)
            if  same_image and same_mask:
                return True
        return False

    def has_key(self, key):
        return self.__contains__(key)

    def clear(self):
        self._keypackages = list()

    def _fingerprint(self, image, mask):
        """Return the image descriptors and keypoints."""
        image_gray = self._get_grayscale(image)
        image_blurred = self._proportional_gaussian(image_gray)
        mask_gray = self._get_grayscale(mask) if mask is not None else None
        #create and return the fingerprint
        keypoints = self._detector.detect(image_blurred, mask_gray)
        keypoints, descriptors = self._extractor.compute(image_blurred, keypoints)
        return self._Fingerprint(keypoints, descriptors)

    def _get_grayscale(self, image):
        """Convert BGR to grayscale or do nothing if image is already grayscale."""
        try:
            channels = image.shape[2] #i.e. rows, columns, channels <----
        except IndexError:
            channels = 1
        if channels == 1:<commit_msg>Used loop to search for key in keypackages for the Nth time so decided to factor it out to _index_of_key_in_keypackages.<commit_after># -*- coding: utf-8 -*-
"""
imagedict
~~~~~~~~~

Emulates the basic behavior of a dict but with images as keys.

:copyright: (c) 2013 by <NAME>.
:license: MIT, see LICENSE for more details.
"""
from collections import namedtuple

import cv2
import numpy as np


class ImageDict(object):
    _KeyPackage = namedtuple("_KeyPackage", ('image', 'mask', 'fingerprint', 'value'))
    _Fingerprint = namedtuple("_Fingerprint", ('keypoints', 'descriptors'))

    def __init__(self):
        self._keypackages = list()
        self._detector = cv2.FastFeatureDetector()
        self._extractor = cv2.DescriptorExtractor_create('BRISK')
        self._matcher = cv2.BFMatcher(cv2.NORM_L2SQR)

    def __len__(self):
        return len(self._keypackages)

    def __setitem__(self, key, value):
        """Set the given key and value.

        Allowed formats for the key are specified in the parse method
        Allowed formats for image and mask are specified in validate method

        Warning: providing a mask of None is valid, so make sure you actually
        have an image in the mask if you intend to use one.

        Note: a lot of validation is done to avoid confusing errors arising later

        """
        image, mask = self._parse_key_arg(key)
        self._validate_image_and_mask(image, mask)
        #if everything is ok, then create and store the package
        fingerprint = self._fingerprint(image, mask)
        new_keypackage = self._KeyPackage(image, mask, fingerprint, value)
        #overwrite the key if it exists
        try:
            i = self._index_of_key_in_keypackages(image, mask)
            self._keypackages[i] = new_keypackage
            return #finished if a key is overwritten
        except KeyError:
            self._keypackages.append(new_keypackage)

    def get(self, key, d = None):
        """Get the value for the given key or provide the default value."""
        try:
            return self.__getitem__(key)
        except KeyError:
            return d

    def setdefault(self, key, d):
        """D.setdefault(k,d) -> D.get(k,d), also set D[k]=d if k not in D

        Due to the keys being images, the default is required as there is no
        standard meaning for a default image.
        """
        try:
            value = self.__getitem__(key)
        except KeyError:
            self.__setitem__(key, d)
            value = d
        return value


    def __getitem__(self, key):
        """Get the value for the given key or raise KeyError if not found.

        Allowed formats for the key are specified in the parse method
        Allowed formats for image and mask are specified in validate method

        Warning: providing a mask of None is valid, so make sure you actually
        have an image in the mask if you intend to use one.

        Note: a lot of validation is done to avoid confusing errors arising later

        """
        MINIMUM_GOOD_MATCHES = 3
        MINIMUM_MATCH_QUALITY = 0.1

        scene_image, scene_mask = self._parse_key_arg(key)
        self._validate_image_and_mask(scene_image, scene_mask)
        scene_keypoints, scene_descriptors = self._fingerprint(scene_image, scene_mask)
        if not self._keypackages:
            raise KeyError('No keys in this image dictionary.')
        #precalculate some scene properties
        scene_w, scene_h = scene_image.shape[0:2]
        scene_corners = np.float32([(0,0), (scene_w, 0), (scene_w, scene_h), (0, scene_h)])
        #for each key, decide if it has a good match in the scene
        matching_keypackages_with_quality = list()
        for kp in self._keypackages:
            obj_keypoints, obj_descriptors = kp.fingerprint
            matches = self._matcher.match(obj_descriptors, scene_descriptors)
            if len(matches) < MINIMUM_GOOD_MATCHES:
                continue
            #do some filtering of the matches to find the best ones
            distances = [match.distance for match in matches]
            min_dist = min(distances)
            avg_dist = sum(distances) / len(distances)
            min_dist = min_dist or avg_dist * 0.01 #if min_dist is zero, use a small percentage of avg instead
            good_matches = [match for match in matches if match.distance <= 3 * min_dist]
            #stop this keypackage if there are not enough matched points
            if len(good_matches) < MINIMUM_GOOD_MATCHES:
                continue
            #get the corresponding match positions
            obj_matched_points = np.array([obj_keypoints[match.queryIdx].pt for match in good_matches])
            scene_matched_points = np.array([scene_keypoints[match.trainIdx].pt for match in good_matches])
            #make a homography that describes how the key is oriented in the scene (rotated, scaled, etc.)
            homography, homography_inliers_mask = cv2.findHomography(obj_matched_points, scene_matched_points, cv2.RANSAC, 2.0) #2.0: should be very close
            inliers, total_good_matches = np.sum(homography_inliers_mask), len(homography_inliers_mask)
            #find the actual shape of the found key in the scene
            obj_w, obj_h = kp.image.shape[0:2]
            obj_corners = np.float32([(0,0), (obj_w, 0), (obj_w, obj_h), (0, obj_h)])
            found_obj_corners = cv2.perspectiveTransform(obj_corners.reshape(1, -1, 2), homography).reshape(-1, 2)
            #calculate the quality of the image match and keep/reject this keypackage
            quality = self._match_quality(obj_corners, found_obj_corners, inliers, total_good_matches)
            if quality < MINIMUM_MATCH_QUALITY:
                continue #finished with this key
            #store this keypackage and the quality of the match
            matching_keypackages_with_quality.append((kp, quality))
        #choose the best match from all qualifying matches
        try:
            best_kp, best_quality = max(matching_keypackages_with_quality, key=lambda x: x[1])
        except ValueError:
            raise KeyError('No good match found.')
        return best_kp.value

    def __delitem__(self, key):
        """Delete the provided key. Raise KeyError if it doesn't exist."""
        image, mask = self._parse_key_arg(key)
        self._validate_image_and_mask(image, mask)
        #find and delete the key
        try:
            i = self._index_of_key_in_keypackages(image, mask)
            del(self._keypackages[i])
            return
        except KeyError:
            raise KeyError('The provided key was not found.')

    def __iter__(self):
        """Provide an iterator over the keys."""
        for kp in self._keypackages:
            yield (kp.image, kp.mask)

    def iterkeys(self):
        return self.__iter__()

    def keys(self):
        return list(self.__iter__())

    def itervalues(self):
        for kp in self._keypackages:
            yield kp.value

    def values(self):
        return list(self.itervalues())

    def iteritems(self):
        for kp in self._keypackages:
            yield ((kp.image, kp.mask), kp.value)

    def items(self):
        return list(self.iteritems())

    def __contains__(self, key):
        """Return True if the image, mask combination is present."""
        image, mask = self._parse_key_arg(key)
        self._validate_image_and_mask(image, mask)
        try:
            index = self._index_of_key_in_keypackages(image, mask)
            return True #if index exists, then contains is true
        except KeyError:
            return False

    def has_key(self, key):
        return self.__contains__(key)

    def clear(self):
        self._keypackages = list()

    def _index_of_key_in_keypackages(self, image, mask):
        """Find the index of the validated key or raise KeyError if not found."""
        for i, kp in enumerate(self._keypackages):
            same_image = np.all(image == kp.image)
            same_mask = np.all(mask == kp.mask)
            if  same_image and same_mask:
                return i
        raise KeyError('Key not found in dictionary.')

    def _fingerprint(self, image, mask):
        """Return the image descriptors and keypoints."""
        image_gray = self._get_grayscale(image)
        image_blurred = self._proportional_gaussian(image_gray)
        mask_gray = self._get_grayscale(mask) if mask is not None else None
        #create and return the fingerprint
        keypoints = self._detector.detect(image_blurred, mask_gray)
        keypoints, descriptors = self._extractor.compute(image_blurred, keypoints)
        return self._Fingerprint(keypoints, descriptors)

    def _get_grayscale(self, image):
        """Convert BGR to grayscale or do nothing if image is already grayscale."""
        try:
            channels = image.shape[2] #i.e. rows, columns, channels <----
        except IndexError:
            channels = 1
        if channels == 1:
{'fraction_non_alphanumeric': 0.08192281651997292, 'fraction_numerical': 0.005642067253441661, 'mean_word_length': 2.4693677823448814, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3336217', 'n_tokens_mistral': 5297, 'n_tokens_neox': 4935, 'n_words': 1647}