<commit_before>import logging
import os
import re

import subprocess
from concurrent import futures

from spreads.plugin import HookPlugin

logger = logging.getLogger('spreadsplug.tesseract')


class TesseractPlugin(HookPlugin):
    @classmethod
    def add_arguments(cls, command, parser):
        if command == 'postprocess':
            parser.add_argument("--language", "-l",
                                dest="language", action="store_true",
                                help="OCR language (3-letter language code")

    def __init__(self, config):
        self.config = config['postprocess']

    def process(self, path):
        logger.info("Performing OCR")
        img_dir = os.path.join(path, 'done')
        with futures.ProcessPoolExecutor() as executor:
            for img in os.listdir(img_dir):
                if not img.endswith('tif'):
                    continue
                executor.submit(
                    subprocess.check_call,
                    ["tesseract", os.path.join(img_dir, img),
                     os.path.join(img_dir, os.path.splitext(img)[0]),
                     "-l", "fra", "hocr"],
                    stderr=subprocess.STDOUT
                )
        # NOTE: This modifies the hOCR files to make them compatible with
        #       pdfbeads.
        #       See the following bugreport for more information:
        #       http://rubyforge.org/tracker/index.php?func=detail&aid=29737&group_id=9752&atid=37737
        for hocr in os.listdir(img_dir):
            if not hocr.endswith('html'):
                continue
            fname = os.path.join(img_dir, hocr)
            with open(fname, 'r') as fp:
                new_content = re.sub(r'(<span[^>]*>(<strong>)? +(<\/strong>)?'
                                     r'<\/span>*)(<span[^>]*>(<strong>)? '
                                     r'+(<\/strong>)?<\/span> *)',
                                     r'\g<1>', fp.read())
            with open(fname, 'w') as fp:
                fp.write(new_content)<commit_msg>tesseract: Combine hOCR files into a single file during output stage<commit_after>import logging
import os
import re
import subprocess
import xml.etree.cElementTree as ET

from concurrent import futures

from spreads.plugin import HookPlugin

logger = logging.getLogger('spreadsplug.tesseract')


class TesseractPlugin(HookPlugin):
    @classmethod
    def add_arguments(cls, command, parser):
        if command == 'postprocess':
            parser.add_argument("--language", "-l",
                                dest="language", action="store_true",
                                help="OCR language (3-letter language code")

    def __init__(self, config):
        self.config = config['postprocess']

    def process(self, path):
        logger.info("Performing OCR")
        img_dir = os.path.join(path, 'done')
        with futures.ProcessPoolExecutor() as executor:
            for img in os.listdir(img_dir):
                if not img.endswith('tif'):
                    continue
                executor.submit(
                    subprocess.check_call,
                    ["tesseract", os.path.join(img_dir, img),
                     os.path.join(img_dir, os.path.splitext(img)[0]),
                     "-l", "fra", "hocr"],
                    stderr=subprocess.STDOUT
                )
        # NOTE: This modifies the hOCR files to make them compatible with
        #       pdfbeads.
        #       See the following bugreport for more information:
        #       http://rubyforge.org/tracker/index.php?func=detail&aid=29737&group_id=9752&atid=37737
        for hocr in os.listdir(img_dir):
            if not hocr.endswith('html'):
                continue
            fname = os.path.join(img_dir, hocr)
            with open(fname, 'r') as fp:
                new_content = re.sub(r'(<span[^>]*>(<strong>)? +(<\/strong>)?'
                                     r'<\/span>*)(<span[^>]*>(<strong>)? '
                                     r'+(<\/strong>)?<\/span> *)',
                                     r'\g<1>', fp.read())
            with open(fname, 'w') as fp:
                fp.write(new_content)

    def output(self, path):
        outfile = os.path.join(path, 'out', "{0}.hocr"
                                            .format(os.path.basename(path)))
        path = os.path.join(path, 'done')
        out_root = ET.Element('html')
        ET.SubElement(out_root, 'head')
        body = ET.SubElement(out_root, 'body')

        for idx, page in enumerate(sorted([os.path.join(path, x)
                                   for x in os.listdir(path)
                                   if x.endswith('.html')])):
            # NOTE: Fixe some things from hOCR output so we can parse it...
            with open(page, 'r+') as fp:
                content = re.sub(r'<em><\/em>', '', fp.read())
                content = re.sub(r'<strong><\/strong>', '', content)
            page = ET.fromstring(content)
            page_elem = page.find("xhtml:body/xhtml:div[@class='ocr_page']",
                                  dict(xhtml="http://www.w3.org/1999/xhtml"))
            # Correct page_number
            page_elem.set('id', 'page_{0}'.format(idx))

            # And tack it onto the output file
            if page_elem:
                body.append(page_elem)
        with open(outfile, 'w') as fp:
            # Strip those annoying namespace tags...
            out_string = re.sub(r"(<\/*)html:", "\g<1>",
                                ET.tostring(out_root))
            fp.write(out_string.encode("UTF-8"))
{'fraction_non_alphanumeric': 0.13122090941639813, 'fraction_numerical': 0.00769781596849266, 'mean_word_length': 1.626704278326281, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2453756', 'n_tokens_mistral': 1662, 'n_tokens_neox': 1560, 'n_words': 344}