<commit_before>from memoized_property import memoized_property
from datalake_common import DatalakeRecord
from datalake_common.errors import InsufficientConfiguration, \
    UnsupportedTimeRange
from translator import S3ToDatalakeTranslator
import time
import logging
from storage import DynamoDBStorage
from queue import SQSQueue
from reporter import SNSReporter
from errors import InvalidS3Notification, InvalidS3Event


logger = logging.getLogger('ingester')


# This is a list of exceptions that we may encounter that do not compromise our
# ability to ingest. These we simply wish to log, report, and move on.
SAFE_EXCEPTIONS = [
    InvalidS3Notification,
    InvalidS3Event,
    UnsupportedTimeRange,
]


class IngesterReport(dict):
<commit_msg>tolerate NoSuchDatalakeFile exception in ingester<commit_after>from memoized_property import memoized_property
from datalake_common import DatalakeRecord
from datalake_common.errors import InsufficientConfiguration, \
    UnsupportedTimeRange, NoSuchDatalakeFile
from translator import S3ToDatalakeTranslator
import time
import logging
from storage import DynamoDBStorage
from queue import SQSQueue
from reporter import SNSReporter
from errors import InvalidS3Notification, InvalidS3Event


logger = logging.getLogger('ingester')


# This is a list of exceptions that we may encounter that do not compromise our
# ability to ingest. These we simply wish to log, report, and move on.
SAFE_EXCEPTIONS = [
    InvalidS3Notification,
    InvalidS3Event,
    UnsupportedTimeRange,
    NoSuchDatalakeFile,
]


class IngesterReport(dict):

{'fraction_non_alphanumeric': 0.036616161616161616, 'fraction_numerical': 0.006313131313131313, 'mean_word_length': 5.9823788546255505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2859131', 'n_tokens_mistral': 441, 'n_tokens_neox': 425, 'n_words': 154}