<commit_before>import logging
import shlex
import time

from django.db import models
from django.utils.encoding import smart_str, smart_unicode
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.db import IntegrityError
from djangae.db import transaction

from google.appengine.ext import db
from google.appengine.ext import deferred

"""
    REMAINING TO DO!

    1. Partial matches. These should be recorded as new Index instances, with an FK to the full term being indexes.
       Partials should only be recorded for between 4, and len(original_term) - 1 characters. Partial matches should be much more highly scored,
       the lower the match, the more the score should be
    2. Cross-join indexing  e.g. book__title on an Author.
    3. Field matches. e.g "id:1234 field1:banana". This should match any other words using indexes, but only return matches that match the field lookups
"""

QUEUE_FOR_INDEXING = getattr(settings, "QUEUE_FOR_INDEXING", "default")

def _do_index(instance, fields_to_index):
    def get_data_from_field(field_, instance_):
        lookups = field_.split("__")
        value = instance
        for lookup in lookups:
            if value is None:
                continue
            value = getattr(value, lookup)

            if "RelatedManager" in value.__class__.__name__:
                if lookup == lookups[-2]:
                    return [ getattr(x, lookups[-1]) for x in value.all() ]
                else:
                    raise TypeError("You can only index one level of related object")

            elif hasattr(value, "__iter__"):
                if lookup == lookups[-1]:
                    return value
                else:
                    raise TypeError("You can only index one level of iterable")

        return [ value ]

    try:
        instance = instance.__class__.objects.get(pk=instance.pk)
    except ObjectDoesNotExist:
        logging.info("Attempting to retrieve object of class: '%s' - with pk: '%s'") % (instance.__class__.__name__, instance.pk)
        raise


    for field in fields_to_index:
        texts = get_data_from_field(field, instance)
        for text in texts:
            if text is None:
                continue

            text = smart_unicode(text)
            text = text.lower() #Normalize<commit_msg>Fix the recently added logging statement<commit_after>import logging
import shlex
import time

from django.db import models
from django.utils.encoding import smart_str, smart_unicode
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.db import IntegrityError
from djangae.db import transaction

from google.appengine.ext import db
from google.appengine.ext import deferred

"""
    REMAINING TO DO!

    1. Partial matches. These should be recorded as new Index instances, with an FK to the full term being indexes.
       Partials should only be recorded for between 4, and len(original_term) - 1 characters. Partial matches should be much more highly scored,
       the lower the match, the more the score should be
    2. Cross-join indexing  e.g. book__title on an Author.
    3. Field matches. e.g "id:1234 field1:banana". This should match any other words using indexes, but only return matches that match the field lookups
"""

QUEUE_FOR_INDEXING = getattr(settings, "QUEUE_FOR_INDEXING", "default")

def _do_index(instance, fields_to_index):
    def get_data_from_field(field_, instance_):
        lookups = field_.split("__")
        value = instance
        for lookup in lookups:
            if value is None:
                continue
            value = getattr(value, lookup)

            if "RelatedManager" in value.__class__.__name__:
                if lookup == lookups[-2]:
                    return [ getattr(x, lookups[-1]) for x in value.all() ]
                else:
                    raise TypeError("You can only index one level of related object")

            elif hasattr(value, "__iter__"):
                if lookup == lookups[-1]:
                    return value
                else:
                    raise TypeError("You can only index one level of iterable")

        return [ value ]

    try:
        instance = instance.__class__.objects.get(pk=instance.pk)
    except ObjectDoesNotExist:
        logging.info("Attempting to retrieve object of class: '%s' - with pk: '%s'", instance.__class__.__name__, instance.pk)
        raise


    for field in fields_to_index:
        texts = get_data_from_field(field, instance)
        for text in texts:
            if text is None:
                continue

            text = smart_unicode(text)
            text = text.lower() #Normalize
{'fraction_non_alphanumeric': 0.07154953429297206, 'fraction_numerical': 0.00550381033022862, 'mean_word_length': 2.4313725490196076, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3914241', 'n_tokens_mistral': 1279, 'n_tokens_neox': 1221, 'n_words': 470}