<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Module for feature extraction from digital images."""

import collections
import itertools
import math

import numpy as np
import cv2

# Color spaces
CS_BGR = 101
CS_HSV = 102
CS_LUV = 103

# Color space ranges used in OpenCV. Each upper boundary is exclusive.
CS_RANGE = {
    CS_BGR: ([0, 256], [0, 256], [0, 256]),
    CS_HSV: ([0, 180], [0, 256], [0, 256]),
    CS_LUV: ([0, 101], [-134, 221], [-140, 123])
}

def segment(img, iters=5, margin=5):
    """Segment image into foreground and background pixels.

    Runs the GrabCut algorithm for segmentation. Returns an 8-bit
    single-channel mask. Its elements may have one of following values:
        * ``cv2.GC_BGD`` defines an obvious background pixel.
        * ``cv2.GC_FGD`` defines an obvious foreground pixel.
        * ``cv2.GC_PR_BGD`` defines a possible background pixel.
        * ``cv2.GC_PR_FGD`` defines a possible foreground pixel.

    The GrabCut algorithm is executed with `iters` iterations. The ROI is set
    to the entire image, with a margin of `margin` pixels from the edges.
    """
    mask = np.zeros(img.shape[:2], np.uint8)
    bgdmodel = np.zeros((1,65), np.float64)
    fgdmodel = np.zeros((1,65), np.float64)
    rect = (margin, margin, img.shape[1]-margin*2, img.shape[0]-margin*2)
    cv2.grabCut(img, mask, rect, bgdmodel, fgdmodel, iters, cv2.GC_INIT_WITH_RECT)
    return mask

def split_by_mask(img, mask):
    """Split an image into rectangular segments by binary mask.

    Returns an iterator object which returns each segment.
    """
    if img.shape[:2] != mask.shape[:2]:
        raise ValueError("All the input arrays must have same shape")
    if len(mask.shape) != 2:
        raise ValueError("Mask must be binary")

    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    for contour in contours:
        bin_mask = np.zeros(img.shape[:2], dtype=np.uint8)
        cv2.drawContours(bin_mask, [contour], 0, 255, -1, 8, hierarchy, 0)
        output = cv2.bitwise_and(img, img, mask=bin_mask)
        x,y,w,h = cv2.boundingRect(contour)
        yield output[y:y+h, x:x+w]

def color_histograms(img, histsize=None, mask=None, colorspace=CS_BGR):
    """Convenience wrapper for :cv2:`calcHist`.

    Returns the histogram for each channel in `img`.

    The input image colorspace must be set with `colorspace`. Value can be
    one of the following:
        * ``CS_BGR`` for the BGR color space.
        * ``CS_HSV`` for the HSV color space.
        * ``CS_LUV`` for the CIE 1976 (L\*, u\*, v\*) color space.

    Default value is ``CS_BGR``.

    .. note::

        OpenCV uses the following color space ranges:
            * BGR: B=0-255, G=0-255, R=0-255
            * HSV: H=0-179, S=0-255, V=0-255
            * LUV: L=0-100, U=-134-220, V=-140-122

        These ranges are also set in ``CS_RANGE``.
    """
    if histsize and len(histsize) != img.ndim:
        raise ValueError("Argument `histsize` must have `img.ndim` elements.")
    if colorspace not in CS_RANGE:
        raise ValueError("Unknown colorspace.")

    hists = []
    for ch in range(img.ndim):
        if histsize == None:
            bins = abs(CS_RANGE[colorspace][ch][1] - CS_RANGE[colorspace][ch][0])
        else:
            bins = histsize[ch]

        ranges = CS_RANGE[colorspace][ch]
        hist = cv2.calcHist([img], [ch], mask, [bins], ranges)
        hists.append(hist)

    return hists

def unrotate(src, dst, bin_mask):
    """Unrotate an image using the binary mask.
    """
    # Get the center.
    m = cv2.moments(bin_mask, True)
    center = ( int(m['m10']/m['m00']) , int(m['m01']/m['m00']) )

    # Calculate the overall shift.
    outlines = simple_outline(bin_mask)
    shift = 0
    for left,right in outlines[1]:
        shift += right - left

    return shift

    # TODO: Get the rotation in degrees.
    # Compare west side and east side areas?
    angle = None

    # TODO: Rotate a contour to get the optimal angle.
    #   For each point(x,y), multiply mat([x\\y\\1]) by rotation matrix and
    #   convert the result back to point coordinates.

    # TODO: Get rotation matrix.
    affine_transform = cv2.getRotationMatrix2D(center, angle, scale)

    # TODO: Rotate the image.
    rotated = cv2.warpAffine(img, affine_transform, dsize)

def get_largest_countour(img, mode, method):
    """Get the largest contour from a binary image.

    It is a simple wrapper for ref:`cv2.findContours`.
    """
    if len(img.shape) != 2:
        raise ValueError("Input image must be binary")

    contours, hierarchy = cv2.findContours(img, mode, method)
    if len(contours) == 1:
        return contours[0]

    largest = None
    area_max = 0
    for i, contour in enumerate(contours):
        area = cv2.contourArea(contour, oriented=False)
        if area > area_max:
            area_max = area<commit_msg>Removed unused function<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Module for feature extraction from digital images."""

import collections
import itertools
import math

import numpy as np
import cv2

# Color spaces
CS_BGR = 101
CS_HSV = 102
CS_LUV = 103

# Color space ranges used in OpenCV. Each upper boundary is exclusive.
CS_RANGE = {
    CS_BGR: ([0, 256], [0, 256], [0, 256]),
    CS_HSV: ([0, 180], [0, 256], [0, 256]),
    CS_LUV: ([0, 101], [-134, 221], [-140, 123])
}

def segment(img, iters=5, margin=5):
    """Segment image into foreground and background pixels.

    Runs the GrabCut algorithm for segmentation. Returns an 8-bit
    single-channel mask. Its elements may have one of following values:
        * ``cv2.GC_BGD`` defines an obvious background pixel.
        * ``cv2.GC_FGD`` defines an obvious foreground pixel.
        * ``cv2.GC_PR_BGD`` defines a possible background pixel.
        * ``cv2.GC_PR_FGD`` defines a possible foreground pixel.

    The GrabCut algorithm is executed with `iters` iterations. The ROI is set
    to the entire image, with a margin of `margin` pixels from the edges.
    """
    mask = np.zeros(img.shape[:2], np.uint8)
    bgdmodel = np.zeros((1,65), np.float64)
    fgdmodel = np.zeros((1,65), np.float64)
    rect = (margin, margin, img.shape[1]-margin*2, img.shape[0]-margin*2)
    cv2.grabCut(img, mask, rect, bgdmodel, fgdmodel, iters, cv2.GC_INIT_WITH_RECT)
    return mask

def split_by_mask(img, mask):
    """Split an image into rectangular segments by binary mask.

    Returns an iterator object which returns each segment.
    """
    if img.shape[:2] != mask.shape[:2]:
        raise ValueError("All the input arrays must have same shape")
    if len(mask.shape) != 2:
        raise ValueError("Mask must be binary")

    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    for contour in contours:
        bin_mask = np.zeros(img.shape[:2], dtype=np.uint8)
        cv2.drawContours(bin_mask, [contour], 0, 255, -1, 8, hierarchy, 0)
        output = cv2.bitwise_and(img, img, mask=bin_mask)
        x,y,w,h = cv2.boundingRect(contour)
        yield output[y:y+h, x:x+w]

def color_histograms(img, histsize=None, mask=None, colorspace=CS_BGR):
    """Convenience wrapper for :cv2:`calcHist`.

    Returns the histogram for each channel in `img`.

    The input image colorspace must be set with `colorspace`. Value can be
    one of the following:
        * ``CS_BGR`` for the BGR color space.
        * ``CS_HSV`` for the HSV color space.
        * ``CS_LUV`` for the CIE 1976 (L\*, u\*, v\*) color space.

    Default value is ``CS_BGR``.

    .. note::

        OpenCV uses the following color space ranges:
            * BGR: B=0-255, G=0-255, R=0-255
            * HSV: H=0-179, S=0-255, V=0-255
            * LUV: L=0-100, U=-134-220, V=-140-122

        These ranges are also set in ``CS_RANGE``.
    """
    if histsize and len(histsize) != img.ndim:
        raise ValueError("Argument `histsize` must have `img.ndim` elements.")
    if colorspace not in CS_RANGE:
        raise ValueError("Unknown colorspace.")

    hists = []
    for ch in range(img.ndim):
        if histsize == None:
            bins = abs(CS_RANGE[colorspace][ch][1] - CS_RANGE[colorspace][ch][0])
        else:
            bins = histsize[ch]

        ranges = CS_RANGE[colorspace][ch]
        hist = cv2.calcHist([img], [ch], mask, [bins], ranges)
        hists.append(hist)

    return hists

def get_largest_countour(img, mode, method):
    """Get the largest contour from a binary image.

    It is a simple wrapper for ref:`cv2.findContours`.
    """
    if len(img.shape) != 2:
        raise ValueError("Input image must be binary")

    contours, hierarchy = cv2.findContours(img, mode, method)
    if len(contours) == 1:
        return contours[0]

    largest = None
    area_max = 0
    for i, contour in enumerate(contours):
        area = cv2.contourArea(contour, oriented=False)
        if area > area_max:
            area_max = area
{'fraction_non_alphanumeric': 0.12516703786191538, 'fraction_numerical': 0.03485523385300668, 'mean_word_length': 2.8396750748182984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3057875', 'n_tokens_mistral': 3194, 'n_tokens_neox': 2957, 'n_words': 936}