<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  IkaLog
#  ======
#  Copyright (C) 2015-2016 <NAME>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import cv2
import numpy as np
import traceback

from ikalog.utils.find_image_file import find_image_file
from ikalog.utils.ikautils import IkaUtils
from ikalog.utils.image_filters.filters import *

from ikalog.utils.ikamatcher2.reference import Numpy_uint8_fast

default_kernel = Numpy_uint8_fast


class IkaMatcher2(object):

    zeros128 = np.array([0] * 128, dtype=np.uint8)

    def _is_cropped(self, img):
        """
        Check the image size and check if the image is already cropped.
        """
        cropped = \
            (img.shape[0] == self._height) and \
            (img.shape[1] == self._width)
        return cropped

    def get_img_object(self, img):
        if not self._is_cropped(img):
            img = img[self._top: self._top + self._height,
                      self._left: self._left + self._width]

        if len(img.shape) == 2:
            img_gray = img
            img_bgr = None
        else:
            img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
            img_bgr = img

        return { 'bgr': img_bgr, 'gray': img_gray, 'bg': None, 'fg': None }

    def match(self, img, debug=None):
        matched, fg_score, bg_score = self.match_score(img, debug)
        return matched

    def match_score(self, img, debug=None):
        img_obj = self.get_img_object(img)
        return self.match_score_internal(img_obj, debug=debug)

    def match_score_internal(self, img_obj, debug=None):
        debug = debug or self._debug

        bg_pixels = 0
        bg_ratio = 0.0
        bg_matched = False

        fg_pixels = 0
        fg_ratio = 0.0
        fg_matched = False

        # Phase 2: Background check
        try:
            if img_obj['bg'] is None:
                img_bg = 255 - \
                    self._bg_method(img_bgr=img_obj['bgr'], img_gray=img_obj['gray'])
                img_obj['bg'] = self._kernel.encode(img_bg)
            bg_pixels = self._kernel.logical_and_popcnt(img_obj['bg'])

            # fixme: zero division
            bg_ratio = bg_pixels / (self._width * self._height)
            bg_matched = (bg_ratio <= self._orig_threshold)
        except:
            IkaUtils.dprint('%s (%s): bg_method %s caused a exception.' % (
                self, self._label, self._bg_method.__class__.__name__))
            IkaUtils.dprint(traceback.format_exc())

        # Phase 3: Foreground check
        if bg_matched:
            if img_obj['fg'] is None:
                img_fg = self._fg_method(img_bgr=img_obj['bgr'], img_gray=img_obj['gray'])
                img_obj['fg'] = self._kernel.encode(img_fg)
            fg_pixels = self._kernel.logical_or_popcnt(img_obj['fg'])

            # fixme: zero division
            fg_ratio = fg_pixels / (self._width * self._height)
            fg_matched = (fg_ratio > self._threshold)

        if debug:
            label = self._label
            print("%s: result=%s raito BG %s FG %s (threshold BG %1.3f FG %1.3f) label:%s" %
                  (self.__class__.__name__, fg_matched, bg_ratio, fg_ratio,<commit_msg>utils/ikamatcher2/matcher: Surpress unnecessary grayscale conversion<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  IkaLog
#  ======
#  Copyright (C) 2015-2016 <NAME>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import cv2
import numpy as np
import traceback

from ikalog.utils.find_image_file import find_image_file
from ikalog.utils.ikautils import IkaUtils
from ikalog.utils.image_filters.filters import *

from ikalog.utils.ikamatcher2.reference import Numpy_uint8_fast

default_kernel = Numpy_uint8_fast


class IkaMatcher2(object):

    zeros128 = np.array([0] * 128, dtype=np.uint8)

    def _is_cropped(self, img):
        """
        Check the image size and check if the image is already cropped.
        """
        cropped = \
            (img.shape[0] == self._height) and \
            (img.shape[1] == self._width)
        return cropped

    def generate_grayscale_image(self, img_obj):
        if img_obj['gray'] is not None:
            return

        img_obj['gray'] = cv2.cvtColor(img_obj['bgr'], cv2.COLOR_BGR2GRAY)

    def get_img_object(self, img):
        if not self._is_cropped(img):
            img = img[self._top: self._top + self._height,
                      self._left: self._left + self._width]

        if len(img.shape) == 2:
            img_gray = img
            img_bgr = None
        else:
            img_gray = None
            img_bgr = img

        return { 'bgr': img_bgr, 'gray': img_gray, 'bg': None, 'fg': None }

    def match(self, img, debug=None):
        matched, fg_score, bg_score = self.match_score(img, debug)
        return matched

    def match_score(self, img, debug=None):
        img_obj = self.get_img_object(img)
        return self.match_score_internal(img_obj, debug=debug)

    def match_score_internal(self, img_obj, debug=None):
        debug = debug or self._debug

        bg_pixels = 0
        bg_ratio = 0.0
        bg_matched = False

        fg_pixels = 0
        fg_ratio = 0.0
        fg_matched = False

        # Phase 2: Background check
        try:
            if img_obj['bg'] is None:
                if self._bg_method.want_grayscale_image and (img_obj['gray'] is None):
                    self.generate_grayscale_image(img_obj)

                img_bg = 255 - \
                    self._bg_method(img_bgr=img_obj['bgr'], img_gray=img_obj['gray'])
                img_obj['bg'] = self._kernel.encode(img_bg)
            bg_pixels = self._kernel.logical_and_popcnt(img_obj['bg'])

            # fixme: zero division
            bg_ratio = bg_pixels / (self._width * self._height)
            bg_matched = (bg_ratio <= self._orig_threshold)
        except:
            IkaUtils.dprint('%s (%s): bg_method %s caused a exception.' % (
                self, self._label, self._bg_method.__class__.__name__))
            IkaUtils.dprint(traceback.format_exc())

        # Phase 3: Foreground check
        if bg_matched:
            if img_obj['fg'] is None:
                if self._fg_method.want_grayscale_image and (img_obj['gray'] is None):
                    self.generate_grayscale_image(img_obj)

                img_fg = self._fg_method(img_bgr=img_obj['bgr'], img_gray=img_obj['gray'])
                img_obj['fg'] = self._kernel.encode(img_fg)
            fg_pixels = self._kernel.logical_or_popcnt(img_obj['fg'])

            # fixme: zero division
            fg_ratio = fg_pixels / (self._width * self._height)
            fg_matched = (fg_ratio > self._threshold)

        if debug:
            label = self._label
            print("%s: result=%s raito BG %s FG %s (threshold BG %1.3f FG %1.3f) label:%s" %
                  (self.__class__.__name__, fg_matched, bg_ratio, fg_ratio,
{'fraction_non_alphanumeric': 0.10455290705006937, 'fraction_numerical': 0.01223357296002018, 'mean_word_length': 2.2972972972972974, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4805187', 'n_tokens_mistral': 2608, 'n_tokens_neox': 2481, 'n_words': 634}