<commit_before>from __future__ import division
from numpy.fft import irfft2, rfft2
import numpy
import itertools
import operator


def subimage(rect, image):
    x, y, w, h = rect
    return image[y:y + h, x:x + w]


def pad_bin_image_to_shape(image, shape):
    """
    Padd image to size :shape: with zeros
    """
    h, w = shape
    ih, iw = image.shape
    assert ih <= h
    assert iw <= w
    if iw < w:
        result = numpy.hstack((image, numpy.zeros((ih, w - iw), bool)))
    else:
        result = image
    if ih < h:
        result = numpy.vstack((result, numpy.zeros((h - ih, w), bool)))
    return result


def sum_2d_images(images):
    it = iter(images)
    result = numpy.copy(next(it))
    for image in it:
        result += image
    return result


OVERLAP_TABLE = {
    16: (16, 8, 4, 2, 1),
    15: (15, 5, 3, 1),
    12: (12, 6, 4, 3, 2, 1),
    10: (10, 5, 2, 1),
    9: (9, 3, 1),
    8: (8, 4, 2, 1),
    6: (6, 3, 2, 1),
    5: (5, 1),
    4: (4, 2, 1),
    3: (3, 1),
    2: (2, 1)
}
# A number near float max which we don't want to get near to keep precision
ACCURACY_LIMIT = 2 ** (64 - 22)


def best_convolution(bin_template, bin_image,
                     tollerance=0.5, overlap_table=OVERLAP_TABLE):
    """
    Selects and applies the best convolution method to find template in image.

    Returns a list of matches in (width, height, x offset, y offset)
    format (where the x and y offsets are from the top left corner).

    As the images are binary images, we can utilise the extra bit space in the
    float64's by cutting the image into tiles and stacking them into variable
    grayscale values.

    This allows converting a sparse binary image into a dense(r) grayscale one.
    """

    template_sum = numpy.count_nonzero(bin_template)
    th, tw = bin_template.shape
    ih, iw = bin_image.shape
    if template_sum == 0 or th == 0 or tw == 0:
        # If we don't have a template
        return []
    if th > ih or tw > iw:
        # If the template is bigger than the image
        return []

    # How many cells can we split the image into?
    max_vert_cells = ih // th
    max_hor_cells = iw // th<commit_msg>Reduced ACCURACY_LIMIT to get rid of bad matches.<commit_after>from __future__ import division
from numpy.fft import irfft2, rfft2
import numpy
import itertools
import operator


def subimage(rect, image):
    x, y, w, h = rect
    return image[y:y + h, x:x + w]


def pad_bin_image_to_shape(image, shape):
    """
    Padd image to size :shape: with zeros
    """
    h, w = shape
    ih, iw = image.shape
    assert ih <= h
    assert iw <= w
    if iw < w:
        result = numpy.hstack((image, numpy.zeros((ih, w - iw), bool)))
    else:
        result = image
    if ih < h:
        result = numpy.vstack((result, numpy.zeros((h - ih, w), bool)))
    return result


def sum_2d_images(images):
    it = iter(images)
    result = numpy.copy(next(it))
    for image in it:
        result += image
    return result


OVERLAP_TABLE = {
    16: (16, 8, 4, 2, 1),
    15: (15, 5, 3, 1),
    12: (12, 6, 4, 3, 2, 1),
    10: (10, 5, 2, 1),
    9: (9, 3, 1),
    8: (8, 4, 2, 1),
    6: (6, 3, 2, 1),
    5: (5, 1),
    4: (4, 2, 1),
    3: (3, 1),
    2: (2, 1)
}

# A number near float max which we don't want to get near to keep precision
# If you get false matches consider reducing this number.
ACCURACY_LIMIT = 2 ** (64 - 23)


def best_convolution(bin_template, bin_image,
                     tollerance=0.5, overlap_table=OVERLAP_TABLE):
    """
    Selects and applies the best convolution method to find template in image.

    Returns a list of matches in (width, height, x offset, y offset)
    format (where the x and y offsets are from the top left corner).

    As the images are binary images, we can utilise the extra bit space in the
    float64's by cutting the image into tiles and stacking them into variable
    grayscale values.

    This allows converting a sparse binary image into a dense(r) grayscale one.
    """

    template_sum = numpy.count_nonzero(bin_template)
    th, tw = bin_template.shape
    ih, iw = bin_image.shape
    if template_sum == 0 or th == 0 or tw == 0:
        # If we don't have a template
        return []
    if th > ih or tw > iw:
        # If the template is bigger than the image
        return []

    # How many cells can we split the image into?
    max_vert_cells = ih // th
    max_hor_cells = iw // th
{'fraction_non_alphanumeric': 0.10850439882697947, 'fraction_numerical': 0.03293480712835552, 'mean_word_length': 2.4081475787855497, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2525453', 'n_tokens_mistral': 1638, 'n_tokens_neox': 1501, 'n_words': 603}