<commit_before>#!/usr/bin/python

import cv2
import numpy as np
from PIL import Image, ImageDraw
from scipy.ndimage.filters import rank_filter
from scipy import stats
import pytesseract
import os
import traceback
import sys

class Page:
    def __init__(self, im, page_num):
        self.healthy = True
        self.err = False
        self.page_num = page_num
        self.orig_im = im
        self.orig_shape = self.orig_im.shape

    def crop(self):
        try:
            self.image, self.num_tries = process_image(self.orig_im)
            self.crop_shape = self.image.shape
            return self.image
        except Exception as e:
            for frame in traceback.extract_tb(sys.exc_info()[2]):
                fname,lineno,fn,text = frame
                print "Error in %s on line %d" % (fname, lineno)
                print e
            self.err = e
            self.healthy = False

    def deskew(self):
        try:
            self.image, self.theta_est = process_skewed_crop(self.image)
            return self.image
        except Exception as e:
            self.err = e
            self.healthy = False

    def extract_text(self):
        temp_path = 'text_temp.png'
        cv2.imwrite(temp_path, self.image)
        self.text = pytesseract.image_to_string(Image.open(temp_path))
        os.remove(temp_path)
        return self.text

    def save(self, out_path):
        if not self.healthy:
            print "There was an error when cropping"
            raise Exception(self.err)
        else:
            self.imwrite(out_path, self.image)


def auto_canny(image, sigma=0.33):
    v = np.median(image)
    lower = int(max(0, (1.0 - sigma) * v))
    upper = int(min(255, (1.0 + sigma) * v))
    edged = cv2.Canny(image, lower, upper, True)
    return edged

def dilate(image, kernel, iterations):
    dilated_image = cv2.dilate(image, kernel, iterations = iterations)
    return dilated_image

def downscale_image(im, max_dim=2048):
    """Shrink im until its longest dimension is <= max_dim.
    Returns new_image, scale (where scale <= 1).
    """
    a, b = im.shape[:2]
    if max(a, b) <= max_dim:
        return 1.0, im

    scale = 1.0 * max_dim / max(a, b)
    new_im = cv2.resize(im, (int(b * scale), int(a * scale)), cv2.INTER_AREA)
    return scale, new_im

def find_components(im, max_components=16):
    """Dilate the image until there are just a few connected components.
    Returns contours for these components."""

    kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(10,10))
    dilation = dilate(im, kernel, 6)

    count = 21
    n = 0
    sigma = 0.000

    while count > max_components:
        n += 1
        sigma += 0.005
        _, contours, hierarchy = cv2.findContours(dilation, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        possible = find_likely_rectangles(contours, sigma)
        count = len(possible)

    return (dilation, possible, n)

def find_likely_rectangles(contours, sigma):
    contours = sorted(contours, key = cv2.contourArea, reverse = True)[:10]
    possible = []
    for c in contours:

        # approximate the contour
        peri = cv2.arcLength(c, True)
        approx = cv2.approxPolyDP(c, sigma * peri, True)
        box = make_box(approx)
        possible.append(box)

    return possible

def make_box(poly):
    x = []
    y = []
    for p in poly:
        for point in p:
            x.append(point[0])
            y.append(point[1])
    xmax = max(x)
    ymax = max(y)
    xmin = min(x)<commit_msg>This actually returns two arguments. Version differences?<commit_after>#!/usr/bin/python

import cv2
import numpy as np
from PIL import Image, ImageDraw
from scipy.ndimage.filters import rank_filter
from scipy import stats
import pytesseract
import os
import traceback
import sys

class Page:
    def __init__(self, im, page_num):
        self.healthy = True
        self.err = False
        self.page_num = page_num
        self.orig_im = im
        self.orig_shape = self.orig_im.shape

    def crop(self):
        try:
            self.image, self.num_tries = process_image(self.orig_im)
            self.crop_shape = self.image.shape
            return self.image
        except Exception as e:
            for frame in traceback.extract_tb(sys.exc_info()[2]):
                fname,lineno,fn,text = frame
                print "Error in %s on line %d" % (fname, lineno)
                print e
            self.err = e
            self.healthy = False

    def deskew(self):
        try:
            self.image, self.theta_est = process_skewed_crop(self.image)
            return self.image
        except Exception as e:
            self.err = e
            self.healthy = False

    def extract_text(self):
        temp_path = 'text_temp.png'
        cv2.imwrite(temp_path, self.image)
        self.text = pytesseract.image_to_string(Image.open(temp_path))
        os.remove(temp_path)
        return self.text

    def save(self, out_path):
        if not self.healthy:
            print "There was an error when cropping"
            raise Exception(self.err)
        else:
            self.imwrite(out_path, self.image)


def auto_canny(image, sigma=0.33):
    v = np.median(image)
    lower = int(max(0, (1.0 - sigma) * v))
    upper = int(min(255, (1.0 + sigma) * v))
    edged = cv2.Canny(image, lower, upper, True)
    return edged

def dilate(image, kernel, iterations):
    dilated_image = cv2.dilate(image, kernel, iterations = iterations)
    return dilated_image

def downscale_image(im, max_dim=2048):
    """Shrink im until its longest dimension is <= max_dim.
    Returns new_image, scale (where scale <= 1).
    """
    a, b = im.shape[:2]
    if max(a, b) <= max_dim:
        return 1.0, im

    scale = 1.0 * max_dim / max(a, b)
    new_im = cv2.resize(im, (int(b * scale), int(a * scale)), cv2.INTER_AREA)
    return scale, new_im

def find_components(im, max_components=16):
    """Dilate the image until there are just a few connected components.
    Returns contours for these components."""

    kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(10,10))
    dilation = dilate(im, kernel, 6)

    count = 21
    n = 0
    sigma = 0.000

    while count > max_components:
        n += 1
        sigma += 0.005
        contours, hierarchy = cv2.findContours(dilation, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        possible = find_likely_rectangles(contours, sigma)
        count = len(possible)

    return (dilation, possible, n)

def find_likely_rectangles(contours, sigma):
    contours = sorted(contours, key = cv2.contourArea, reverse = True)[:10]
    possible = []
    for c in contours:

        # approximate the contour
        peri = cv2.arcLength(c, True)
        approx = cv2.approxPolyDP(c, sigma * peri, True)
        box = make_box(approx)
        possible.append(box)

    return possible

def make_box(poly):
    x = []
    y = []
    for p in poly:
        for point in p:
            x.append(point[0])
            y.append(point[1])
    xmax = max(x)
    ymax = max(y)
    xmin = min(x)
{'fraction_non_alphanumeric': 0.10396814108946095, 'fraction_numerical': 0.016782818944673587, 'mean_word_length': 2.290594291062237, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '299621', 'n_tokens_mistral': 2404, 'n_tokens_neox': 2277, 'n_words': 607}