<commit_before>import threading
import logging

from django.conf import settings

from fluent import cldr_rules
from djangae.db import transaction

from google.appengine.ext.deferred import defer


_state = threading.local()

LANGUAGE_TRANSLATIONS_KEY = "fluent_translations_%s"


def invalidate_language(language_code):
    from django.core.cache import cache
    if hasattr(_state, "translations_dicts") and language_code in _state.translations_dicts:
        del _state.translations_dicts
    cache.delete(LANGUAGE_TRANSLATIONS_KEY % language_code)

def _translation_to_dict(trans):
    data = {"singular": trans.text, "plurals": trans.plural_texts}
    data.update(trans.plural_texts)
    return data

def _load_into_memcache(language_code):
    from django.core.cache import cache
    from .models import Translation

    logging.info("Reloading translations from the database for %s", language_code)
    # Not in cache or local state, so let's query...
    with transaction.non_atomic():
        translations = Translation.objects.filter(language_code=language_code)

    translations_dict = {}
    for t in translations:
        translations_dict[(t.denorm_master_text, t.denorm_master_hint)] = _translation_to_dict(t)

    if not getattr(_state, "translations_dict", None):
        _state.translations_dicts = {}

    _state.translations_dicts[language_code] = translations_dict

    cache.set(LANGUAGE_TRANSLATIONS_KEY % language_code, translations_dict)
    return translations_dict

def _get_translations_dict(language_code, text=None, hint=None):
    """ Returns a dict of all the translations for the given language_code,
        where the key is a tuple of (default, hint) and the value is
        the translated text.
        Tries to get it in this order:
        1. From thread locals
        2. From memcache
        3. From the database.
        Note that it is quicker to fetch all the translations for a language
        in a single query and store them in a dict than it is to do a separate
        query for each translation, hence this approach.

        If you specify text, and hint, the update will be done offline and the result
        of this function will just be a dictionary containing the single translation you
        were after
    """
    from django.core.cache import cache
    from .models import Translation, MasterTranslation

    if not getattr(_state, 'translations_dicts', None):
        _state.translations_dicts = {}

    try:
        return _state.translations_dicts[language_code]
    except KeyError:
        pass

    # Getting from local state failed so lets try memcache
    cached = cache.get(LANGUAGE_TRANSLATIONS_KEY % language_code)
    if cached:
        # Add to the local state, and then return it
        _state.translations_dicts[language_code] = cached
        return cached

    # If we specified the text and hint we were looking for, we defer the cache update
    # and then return that specific translation. Otherwise we do the update inline and
    # return the entire dictionary. Note we only defer the update if the translation was
    # in the database. This is because the language code might not exist, so we'd continually
    # defer tasks for that language
    if text:
        try:
            # we use settings.LANGUAGE_CODE because this should only ever be called for translations
            # in templates or code which always have a master language of settings.LANGUAGE_CODE unlike
            # translatablefields which may have different, but then they have access to the master language
            # code directly. If this assumption is somehow wrong I'm not sure what else to do here...
            master_key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE)
            t = Translation.objects.get(language_code=language_code, master_translation_id=master_key)
            ret = { (text, hint): _translation_to_dict(t) }
        except Translation.DoesNotExist:
            ret = {}
<commit_msg>Again, make sure we aren't querying transactionally<commit_after>import threading
import logging

from django.conf import settings

from fluent import cldr_rules
from djangae.db import transaction

from google.appengine.ext.deferred import defer


_state = threading.local()

LANGUAGE_TRANSLATIONS_KEY = "fluent_translations_%s"


def invalidate_language(language_code):
    from django.core.cache import cache
    if hasattr(_state, "translations_dicts") and language_code in _state.translations_dicts:
        del _state.translations_dicts
    cache.delete(LANGUAGE_TRANSLATIONS_KEY % language_code)

def _translation_to_dict(trans):
    data = {"singular": trans.text, "plurals": trans.plural_texts}
    data.update(trans.plural_texts)
    return data

def _load_into_memcache(language_code):
    from django.core.cache import cache
    from .models import Translation

    logging.info("Reloading translations from the database for %s", language_code)
    # Not in cache or local state, so let's query...
    with transaction.non_atomic():
        translations = Translation.objects.filter(language_code=language_code)

    translations_dict = {}
    for t in translations:
        translations_dict[(t.denorm_master_text, t.denorm_master_hint)] = _translation_to_dict(t)

    if not getattr(_state, "translations_dict", None):
        _state.translations_dicts = {}

    _state.translations_dicts[language_code] = translations_dict

    cache.set(LANGUAGE_TRANSLATIONS_KEY % language_code, translations_dict)
    return translations_dict

def _get_translations_dict(language_code, text=None, hint=None):
    """ Returns a dict of all the translations for the given language_code,
        where the key is a tuple of (default, hint) and the value is
        the translated text.
        Tries to get it in this order:
        1. From thread locals
        2. From memcache
        3. From the database.
        Note that it is quicker to fetch all the translations for a language
        in a single query and store them in a dict than it is to do a separate
        query for each translation, hence this approach.

        If you specify text, and hint, the update will be done offline and the result
        of this function will just be a dictionary containing the single translation you
        were after
    """
    from django.core.cache import cache
    from .models import Translation, MasterTranslation

    if not getattr(_state, 'translations_dicts', None):
        _state.translations_dicts = {}

    try:
        return _state.translations_dicts[language_code]
    except KeyError:
        pass

    # Getting from local state failed so lets try memcache
    cached = cache.get(LANGUAGE_TRANSLATIONS_KEY % language_code)
    if cached:
        # Add to the local state, and then return it
        _state.translations_dicts[language_code] = cached
        return cached

    # If we specified the text and hint we were looking for, we defer the cache update
    # and then return that specific translation. Otherwise we do the update inline and
    # return the entire dictionary. Note we only defer the update if the translation was
    # in the database. This is because the language code might not exist, so we'd continually
    # defer tasks for that language
    if text:
        try:
            # we use settings.LANGUAGE_CODE because this should only ever be called for translations
            # in templates or code which always have a master language of settings.LANGUAGE_CODE unlike
            # translatablefields which may have different, but then they have access to the master language
            # code directly. If this assumption is somehow wrong I'm not sure what else to do here...
            master_key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE)
            with transaction.non_atomic():
                t = Translation.objects.get(language_code=language_code, master_translation_id=master_key)
            ret = { (text, hint): _translation_to_dict(t) }
        except Translation.DoesNotExist:
            ret = {}

{'fraction_non_alphanumeric': 0.05736184942514526, 'fraction_numerical': 0.0007417480529113611, 'mean_word_length': 3.2534174553102, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '57782', 'n_tokens_mistral': 2205, 'n_tokens_neox': 2123, 'n_words': 812}