<commit_before>import os
import logging
import requests
from normality import stringify
from pantomime.types import DEFAULT
from requests import RequestException, HTTPError
from abc import ABC, abstractmethod
from servicelayer import env
from servicelayer.cache import get_redis, make_key
from servicelayer.util import backoff, service_retries

from ingestors.exc import ProcessingException
from ingestors.services.util import ShellCommand
from ingestors.util import join_path, make_directory

log = logging.getLogger(__name__)


class DocumentConverter(ABC):

    @classmethod
    def is_available(cls):
        return False

    def document_to_pdf(self, file_path, result, work_path, archive):
        conn = get_redis()
        key = make_key('pdf', result.checksum)
        if conn.exists(key):
            log.info("Using [%s] PDF from cache", result.file_name)
            pdf_hash = stringify(conn.get(key))
            if pdf_hash is not None:
                return archive.load_file(pdf_hash, temp_path=work_path)

        pdf_file = self._document_to_pdf(file_path, result, work_path)
        content_hash = archive.archive_file(pdf_file)
        result.pdf_checksum = content_hash
        conn.set(key, content_hash)
        return pdf_file

    @abstractmethod
    def _document_to_pdf(self, file_path, result, work_path):
        pass


class LocalDocumentConverter(DocumentConverter, ShellCommand):
    """Provides helpers for Libre/Open Office tools."""

    @classmethod
    def is_available(cls):
        return cls.find_command('soffice') is not None

    def _document_to_pdf(self, file_path, result, work_path):
        """Converts an office document to PDF."""
        instance_dir = make_directory(work_path, 'soffice_instance')
        out_dir = make_directory(work_path, 'soffice_output')
        log.info('Converting [%s] to PDF...', result.file_name)<commit_msg>Restore content hash for cached results<commit_after>import os
import logging
import requests
from normality import stringify
from pantomime.types import DEFAULT
from requests import RequestException, HTTPError
from abc import ABC, abstractmethod
from servicelayer import env
from servicelayer.cache import get_redis, make_key
from servicelayer.util import backoff, service_retries

from ingestors.exc import ProcessingException
from ingestors.services.util import ShellCommand
from ingestors.util import join_path, make_directory

log = logging.getLogger(__name__)


class DocumentConverter(ABC):

    @classmethod
    def is_available(cls):
        return False

    def document_to_pdf(self, file_path, result, work_path, archive):
        conn = get_redis()
        key = make_key('pdf', result.checksum)
        if conn.exists(key):
            content_hash = stringify(conn.get(key))
            log.info("Using [%s] PDF from cache: %s",
                     result.file_name, content_hash)
            if content_hash is not None:
                result.pdf_checksum = content_hash
                return archive.load_file(content_hash, temp_path=work_path)

        pdf_file = self._document_to_pdf(file_path, result, work_path)
        content_hash = archive.archive_file(pdf_file)
        result.pdf_checksum = content_hash
        conn.set(key, content_hash)
        return pdf_file

    @abstractmethod
    def _document_to_pdf(self, file_path, result, work_path):
        pass


class LocalDocumentConverter(DocumentConverter, ShellCommand):
    """Provides helpers for Libre/Open Office tools."""

    @classmethod
    def is_available(cls):
        return cls.find_command('soffice') is not None

    def _document_to_pdf(self, file_path, result, work_path):
        """Converts an office document to PDF."""
        instance_dir = make_directory(work_path, 'soffice_instance')
        out_dir = make_directory(work_path, 'soffice_output')
        log.info('Converting [%s] to PDF...', result.file_name)
{'fraction_non_alphanumeric': 0.0785568065506653, 'fraction_numerical': 0.0, 'mean_word_length': 3.582649472450176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5082883', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1114, 'n_words': 302}