<commit_before>import contextlib

from PyPDF2 import PdfFileReader
from PIL import Image as PILImage
from wand.image import Image
import tesserocr
try:
    import pdflib
except ImportError:
    pdflib = None
    

TESSERACT_LANGUAGE = {
    'en': 'eng',
    'de': 'deu'
}


class PDFProcessor(object):
    def __init__(self, filename, language=None, config=None):
        self.filename = filename
        self.pdf_reader = PdfFileReader(filename)
        self.num_pages = self.pdf_reader.getNumPages()
        self.language = language
        self.config = config or {}

    def get_meta(self):
        doc_info = self.pdf_reader.getDocumentInfo()
        return {
            'title': doc_info.title
        }

    def get_images(self, pages=None, resolution=300):
        if pages is None:
            pages = range(self.num_pages)
        for page_no in pages:
            with self.get_image(page_no, resolution=resolution) as img:
                yield img

    @contextlib.contextmanager
    def get_image(self, page_no, resolution=300):
        filename = "{}[{}]".format(self.filename, page_no)
        with Image(filename=filename, resolution=resolution) as img:
            yield img

    def get_text(self, pages=None):
        if pages is None:
            pages = range(self.num_pages)
        pdflib_pages = None
        if pdflib is not None:
            pdflib_doc = pdflib.Document(self.filename)
            pdflib_pages = list(pdflib_doc)
        for page_no in pages:
            if pdflib_pages is not None:
                page = pages[page_no]
                text = ' '.join(page.lines).strip()
            else:
                page = self.pdf_reader.getPage(page_no)
                text = page.extractText()
            if not text.strip():
                text = self.run_ocr(page_no)
            yield text

    def run_ocr(self, page_no):
        with self.get_image(page_no, resolution=300) as img:
            pil_image = PILImage.frombytes('RGB', img.size, img.make_blob('RGB'))
            return tesserocr.image_to_text(
                pil_image,
                lang=TESSERACT_LANGUAGE[self.language],
                path=self.config.get('TESSERACT_DATA_PATH', '')
            )

    def save_pages(self, path, **kwargs):
        for page, img in enumerate(self.get_images(**kwargs), 1):
            filename = path.format(page=page)
            img.save(filename=filename)
            yield filename


def crop_image(image_path, left, top, width, height):
    with Image(filename=image_path) as img:
        img.crop(left, top, left + width, top + height)
        return img.make_blob('gif')<commit_msg>Fix bug in pdf text extraction<commit_after>import contextlib

from PyPDF2 import PdfFileReader
from PIL import Image as PILImage
from wand.image import Image
import tesserocr
try:
    import pdflib
except ImportError:
    pdflib = None
    

TESSERACT_LANGUAGE = {
    'en': 'eng',
    'de': 'deu'
}


class PDFProcessor(object):
    def __init__(self, filename, language=None, config=None):
        self.filename = filename
        self.pdf_reader = PdfFileReader(filename)
        self.num_pages = self.pdf_reader.getNumPages()
        self.language = language
        self.config = config or {}

    def get_meta(self):
        doc_info = self.pdf_reader.getDocumentInfo()
        return {
            'title': doc_info.title
        }

    def get_images(self, pages=None, resolution=300):
        if pages is None:
            pages = range(self.num_pages)
        for page_no in pages:
            with self.get_image(page_no, resolution=resolution) as img:
                yield img

    @contextlib.contextmanager
    def get_image(self, page_no, resolution=300):
        filename = "{}[{}]".format(self.filename, page_no)
        with Image(filename=filename, resolution=resolution) as img:
            yield img

    def get_text(self, pages=None):
        if pages is None:
            pages = range(self.num_pages)
        pdflib_pages = None
        if pdflib is not None:
            pdflib_doc = pdflib.Document(self.filename)
            pdflib_pages = list(pdflib_doc)
        for page_no in pages:
            if pdflib_pages is not None:
                page = pdflib_pages[page_no]
                text = ' '.join(page.lines).strip()
            else:
                page = self.pdf_reader.getPage(page_no)
                text = page.extractText()
            if not text.strip():
                text = self.run_ocr(page_no)
            yield text

    def run_ocr(self, page_no):
        with self.get_image(page_no, resolution=300) as img:
            pil_image = PILImage.frombytes('RGB', img.size, img.make_blob('RGB'))
            return tesserocr.image_to_text(
                pil_image,
                lang=TESSERACT_LANGUAGE[self.language],
                path=self.config.get('TESSERACT_DATA_PATH', '')
            )

    def save_pages(self, path, **kwargs):
        for page, img in enumerate(self.get_images(**kwargs), 1):
            filename = path.format(page=page)
            img.save(filename=filename)
            yield filename


def crop_image(image_path, left, top, width, height):
    with Image(filename=image_path) as img:
        img.crop(left, top, left + width, top + height)
        return img.make_blob('gif')
{'fraction_non_alphanumeric': 0.09683496608892238, 'fraction_numerical': 0.004144687264506406, 'mean_word_length': 2.175239234449761, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3599515', 'n_tokens_mistral': 1631, 'n_tokens_neox': 1581, 'n_words': 376}