<commit_before>"""
App config for rubber.
"""
from copy import deepcopy
import collections

import logging
import six

from elasticsearch import Elasticsearch

from django.conf import settings
from django.db import transaction
from django.db.models.signals import class_prepared
from django.db.models.signals import post_delete
from django.db.models.signals import post_save

logger = logging.getLogger(__name__)

try:
    from django.apps import AppConfig
except ImportError:
    AppConfig = object


DEFAULT_RUBBER = {
    'HOSTS': ['localhost:9200'],
    'MODELS': [],
    'CONFIG_ROOT': 'es_configs',
    'OPTIONS': {
        'fail_silently': True,
        'disabled': False,
        'celery_queue': None
    },
}


def recursive_update(d, u):
    for k, v in six.iteritems(u):
        if isinstance(v, collections.Mapping):
            r = recursive_update(d.get(k, {}), v)
            d[k] = r
        else:
            d[k] = u[k]
    return d


def post_save_es_index(sender, instance, **kwargs):
    if instance.is_indexable():
        try:
            # post_save fires after the save occurs but before the transaction
            # is commited.
            transaction.on_commit(lambda: instance.es_index())
        except AttributeError:
            # 1s countdown waiting for the transaction to complete.
            instance.es_index(countdown=1)


def post_delete_es_delete(sender, instance, **kwargs):
    instance.es_delete()

<commit_msg>Add error log if es_index is undefined.<commit_after>"""
App config for rubber.
"""
from copy import deepcopy
import collections

import logging
import six

from elasticsearch import Elasticsearch

from django.conf import settings
from django.db import transaction
from django.db.models.signals import class_prepared
from django.db.models.signals import post_delete
from django.db.models.signals import post_save

logger = logging.getLogger(__name__)

try:
    from django.apps import AppConfig
except ImportError:
    AppConfig = object


DEFAULT_RUBBER = {
    'HOSTS': ['localhost:9200'],
    'MODELS': [],
    'CONFIG_ROOT': 'es_configs',
    'OPTIONS': {
        'fail_silently': True,
        'disabled': False,
        'celery_queue': None
    },
}


def recursive_update(d, u):
    for k, v in six.iteritems(u):
        if isinstance(v, collections.Mapping):
            r = recursive_update(d.get(k, {}), v)
            d[k] = r
        else:
            d[k] = u[k]
    return d


def post_save_es_index(sender, instance, **kwargs):
    if instance.is_indexable():
        try:
            # post_save fires after the save occurs but before the transaction
            # is commited.
            transaction.on_commit(lambda: instance.es_index())
        except AttributeError:
            # 1s countdown waiting for the transaction to complete.
            try:
                instance.es_index(countdown=1)
            except AttributeError:
                logger.error(
                    "Sender instance doesn't define es_index.",
                    exc_info=True
                )


def post_delete_es_delete(sender, instance, **kwargs):
    instance.es_delete()


{'fraction_non_alphanumeric': 0.09426751592356689, 'fraction_numerical': 0.003821656050955414, 'mean_word_length': 2.6396292004634994, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2679684', 'n_tokens_mistral': 921, 'n_tokens_neox': 897, 'n_words': 258}