<commit_before>import os
import logging

from ingestors import Manager
from ingestors.util import decode_path
from storagelayer import checksum

from aleph.core import db
from aleph.model import Document, Cache
from aleph.analyze import analyze_document
from aleph.ingest.result import DocumentResult

log = logging.getLogger(__name__)


class DocumentManager(Manager):
    """Handle the process of ingesting documents.

    This includes creating and flushing records, setting document state and
    dispatching child ingestors as needed.
    """

    RESULT_CLASS = DocumentResult

    def __init__(self, config, archive):
        super(DocumentManager, self).__init__(config)
        self.archive = archive

    def before(self, result):
        db.session.flush()
        result.document.status = Document.STATUS_PENDING
        result.document.delete_records()

    def after(self, result):
        result.update()
        db.session.commit()
        if result.error_message:
            log.warn('Error [%r]: %s', result, result.error_message)
        else:
            log.debug('Ingested [%s]: %s',
                      result.document.id,
                      result.document.title)
        analyze_document(result.document)

    def get_cache(self, key):
        return Cache.get_cache(key)

    def set_cache(self, key, value):<commit_msg>Do not double-report ingestion errors<commit_after>import os
import logging

from ingestors import Manager
from ingestors.util import decode_path
from storagelayer import checksum

from aleph.core import db
from aleph.model import Document, Cache
from aleph.analyze import analyze_document
from aleph.ingest.result import DocumentResult

log = logging.getLogger(__name__)


class DocumentManager(Manager):
    """Handle the process of ingesting documents.

    This includes creating and flushing records, setting document state and
    dispatching child ingestors as needed.
    """

    RESULT_CLASS = DocumentResult

    def __init__(self, config, archive):
        super(DocumentManager, self).__init__(config)
        self.archive = archive

    def before(self, result):
        db.session.flush()
        result.document.status = Document.STATUS_PENDING
        result.document.delete_records()

    def after(self, result):
        result.update()
        db.session.commit()
        if not result.error_message:
            log.debug('Ingested [%s]: %s',
                      result.document.id,
                      result.document.title)
        analyze_document(result.document)

    def get_cache(self, key):
        return Cache.get_cache(key)

    def set_cache(self, key, value):
{'fraction_non_alphanumeric': 0.08116270290675727, 'fraction_numerical': 0.0, 'mean_word_length': 2.9493293591654246, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4002052', 'n_tokens_mistral': 724, 'n_tokens_neox': 701, 'n_words': 204}