<commit_before># -*- coding: utf-8 -*-
import time

from django.conf import settings
from django.core.cache import cache
from django.utils import simplejson as json
from django.utils.http import cookie_date
from django.utils.translation import (
    get_language, get_language_info, ugettext as _
)
from django.utils.translation.trans_real import parse_accept_lang_header

from unilangs import get_language_name_mapping, LanguageCode

# A set of all language codes we support.
_supported_languages_map = get_language_name_mapping('unisubs')
_all_languages_map = get_language_name_mapping('internal')
SUPPORTED_LANGUAGE_CODES = set(_supported_languages_map.keys())
ALL_LANGUAGE_CODES = set(_all_languages_map.keys())

SUPPORTED_LANGUAGE_CHOICES = list(sorted(_supported_languages_map.items(),
                                         key=lambda c: c[1]))
ALL_LANGUAGE_CHOICES = list(sorted(_all_languages_map.items(),
                                   key=lambda c: c[1]))

def _only_supported_languages(language_codes):
    """Filter the given list of language codes to contain only codes we support."""

    # TODO: Figure out the codec issue here.
    return [code for code in language_codes if code in SUPPORTED_LANGUAGE_CODES]


def get_language_choices(with_empty=False, with_any=False):
    """Return a list of language code choices labeled appropriately."""

    cache_key = 'simple-langs-cache-%s' % get_language()
    languages = cache.get(cache_key)

    if not languages:
        languages = []

        for code, name in get_language_name_mapping('unisubs').items():
            languages.append((code, _(name)))

        languages.sort(key=lambda item: item[1])
        cache.set(cache_key, languages, 60*60)

    if with_any:
        languages = [('', _('--- Any Language ---'))] + languages

    if with_empty:
        languages = [('', '---------')] + languages

    return languages

def get_language_choices_as_dicts(with_empty=False):
    """Return a list of language code choices labeled appropriately."""
    return [
        {'code': code, 'name': name}
        for (code, name) in get_language_choices(with_empty)
    ]

def get_language_label(code):
    """Return the translated, human-readable label for the given language code."""
    lc = LanguageCode(code, 'unisubs')
    return u'%s' % _(lc.name())


def get_user_languages_from_request(request, readable=False, guess=True):
    """Return a list of our best guess at languages that request.user speaks."""
    languages = []

    if request.user.is_authenticated():
        languages = [l.language for l in request.user.get_languages()]

    if guess and not languages:
        languages = languages_from_request(request)

    if readable:
        return map(get_language_label, _only_supported_languages(languages))
    else:
        return _only_supported_languages(languages)

def set_user_languages_to_cookie(response, languages):
    max_age = 60*60*24
    response.set_cookie(
        settings.USER_LANGUAGES_COOKIE_NAME,
        json.dumps(languages),
        max_age=max_age,
        expires=cookie_date(time.time() + max_age))

def get_user_languages_from_cookie(request):<commit_msg>Fix for get_language_label()<commit_after># -*- coding: utf-8 -*-
import time

from django.conf import settings
from django.core.cache import cache
from django.utils import simplejson as json
from django.utils.http import cookie_date
from django.utils.translation import (
    get_language, get_language_info, ugettext as _
)
from django.utils.translation.trans_real import parse_accept_lang_header

from unilangs import get_language_name_mapping, LanguageCode

# A set of all language codes we support.
_supported_languages_map = get_language_name_mapping('unisubs')
_all_languages_map = get_language_name_mapping('internal')
SUPPORTED_LANGUAGE_CODES = set(_supported_languages_map.keys())
ALL_LANGUAGE_CODES = set(_all_languages_map.keys())

SUPPORTED_LANGUAGE_CHOICES = list(sorted(_supported_languages_map.items(),
                                         key=lambda c: c[1]))
ALL_LANGUAGE_CHOICES = list(sorted(_all_languages_map.items(),
                                   key=lambda c: c[1]))

def _only_supported_languages(language_codes):
    """Filter the given list of language codes to contain only codes we support."""

    # TODO: Figure out the codec issue here.
    return [code for code in language_codes if code in SUPPORTED_LANGUAGE_CODES]


def get_language_choices(with_empty=False, with_any=False):
    """Return a list of language code choices labeled appropriately."""

    cache_key = 'simple-langs-cache-%s' % get_language()
    languages = cache.get(cache_key)

    if not languages:
        languages = []

        for code, name in get_language_name_mapping('unisubs').items():
            languages.append((code, _(name)))

        languages.sort(key=lambda item: item[1])
        cache.set(cache_key, languages, 60*60)

    if with_any:
        languages = [('', _('--- Any Language ---'))] + languages

    if with_empty:
        languages = [('', '---------')] + languages

    return languages

def get_language_choices_as_dicts(with_empty=False):
    """Return a list of language code choices labeled appropriately."""
    return [
        {'code': code, 'name': name}
        for (code, name) in get_language_choices(with_empty)
    ]

def get_language_label(code):
    """Return the translated, human-readable label for the given language code."""
    lc = LanguageCode(code, 'internal')
    return u'%s' % _(lc.name())


def get_user_languages_from_request(request, readable=False, guess=True):
    """Return a list of our best guess at languages that request.user speaks."""
    languages = []

    if request.user.is_authenticated():
        languages = [l.language for l in request.user.get_languages()]

    if guess and not languages:
        languages = languages_from_request(request)

    if readable:
        return map(get_language_label, _only_supported_languages(languages))
    else:
        return _only_supported_languages(languages)

def set_user_languages_to_cookie(response, languages):
    max_age = 60*60*24
    response.set_cookie(
        settings.USER_LANGUAGES_COOKIE_NAME,
        json.dumps(languages),
        max_age=max_age,
        expires=cookie_date(time.time() + max_age))

def get_user_languages_from_cookie(request):
{'fraction_non_alphanumeric': 0.10122602955045583, 'fraction_numerical': 0.0044011317195850365, 'mean_word_length': 3.7168272794662713, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1738004', 'n_tokens_mistral': 1931, 'n_tokens_neox': 1881, 'n_words': 471}