<commit_before>import logging
import os
import sys
import pdb
import pytesseract

from logging import Formatter, FileHandler
from PIL import Image
from PIL import ImageFilter
from PIL import ImageEnhance
from nltk.corpus import words
from StringIO import StringIO

_ALL_WORDS = words.words()

class OcrEngine():
    def process_image(self, im):
        """pass an image PIL object in and run basic text analysis"""
        im.filter(ImageFilter.SHARPEN)
        im2 = self.resize_image(im, 5, 5);im2.filter(ImageFilter.SHARPEN)
        im2.save("temp.jpg");image = Image.open('temp.jpg')
        words_by_row = self._get_rows(pytesseract.image_to_string(image))
        return self._format_output([self._check_group(word_group) for word_group in words_by_row])

    def resize_image(self, image, x, y):
        """resize an image passing in x and y axis multipliers"""
        nx, ny = image.size
        return image.resize((int(nx*x), int(ny*y)), Image.BICUBIC)

    def _get_rows(self, string):
        all_words = string.split("\n")
        last_words = []
        for group in all_words:
            last_words.append(group.split(" "))
        return last_words

    def _check_word(self, word):
        if str(word).lower() in _ALL_WORDS:
            return str(word).lower()
        else:
            return ""

    def _check_group(self, word_group):
        final = []
        for word in word_group:
            final.append(self._check_word(word))
        return final
    def _format_output(self, output):
        final = []
        for group in output:
            for item in group:
                good_group = []
                if item != "":
                    good_group.append(item)
            final.append(good_group)
        return final

ENGINE = OcrEngine()
<commit_msg>comments on some shitty code<commit_after>import logging
import os
import sys
import pdb
import pytesseract

from logging import Formatter, FileHandler
from PIL import Image
from PIL import ImageFilter
from PIL import ImageEnhance
from nltk.corpus import words
from StringIO import StringIO

_ALL_WORDS = words.words()

class OcrEngine():
    def process_image(self, im):
        """pass an image PIL object in and run basic text analysis"""
        im.filter(ImageFilter.SHARPEN)
        im2 = self.resize_image(im, 5, 5);im2.filter(ImageFilter.SHARPEN)
        im2.save("temp.jpg");image = Image.open('temp.jpg')
        words_by_row = self._get_rows(pytesseract.image_to_string(image))
        return self._format_output([self._check_group(word_group) for word_group in words_by_row])

    def resize_image(self, image, x, y):
        """resize an image passing in x and y axis multipliers"""
        nx, ny = image.size
        return image.resize((int(nx*x), int(ny*y)), Image.BICUBIC)

    # FIXME use fucking comprehensions and map
    def _get_rows(self, string):
        all_words = string.split("\n")
        last_words = []
        for group in all_words:
            last_words.append(group.split(" "))
        return last_words

    def _check_word(self, word):
        return str(word).lower() if str(word).lower() in _ALL_WORDS else ""

    # FIXME use fucking comprehensions and map
    def _check_group(self, word_group):
        final = []
        for word in word_group:
            final.append(self._check_word(word))
        return final

    # FIXME use fucking comprehensions and map
    def _format_output(self, output):
        final = []
        for group in output:
            for item in group:
                good_group = []
                if item != "":
                    good_group.append(item)
            final.append(good_group)
        return final

ENGINE = OcrEngine()

{'fraction_non_alphanumeric': 0.09519956850053937, 'fraction_numerical': 0.002696871628910464, 'mean_word_length': 2.566346153846154, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3666495', 'n_tokens_mistral': 1127, 'n_tokens_neox': 1105, 'n_words': 313}