<commit_before>import re

from django.conf import settings
from django.core.urlresolvers import RegexURLPattern, RegexURLResolver, get_callable
from django.core.urlresolvers import NoReverseMatch
from django.core.urlresolvers import get_script_prefix
from django.utils.datastructures import MultiValueDict
from django.utils.encoding import iri_to_uri, force_unicode
from django.utils.regex_helper import normalize
from django.utils.translation import get_language
from django.utils.translation.trans_real import translation

from multilang.settings import LANGUAGE_DOMAINS


def turl(regex, view, kwargs=None, name=None, prefix=''):
    # Copied from django.conf.urls.defaults.url
    if type(view) == list:
        # For include(...) processing.
        return MultilangRegexURLResolver(regex, view[0], kwargs)
    else:
        if isinstance(view, basestring):
            if not view:
                raise ImproperlyConfigured('Empty URL pattern view name not permitted (for pattern %r)' % regex)
            if prefix:
                view = prefix + '.' + view
        return MultilangRegexURLPattern(regex, view, kwargs, name)


def lang_prefixed(urlconf_name):
    return LangSelectionRegexURLResolver(urlconf_name)


_resolvers = {}
def get_resolver(urlconf, lang):
    if urlconf is None:
        from django.conf import settings
        urlconf = settings.ROOT_URLCONF
    key = (urlconf, lang)
    if key not in _resolvers:
        _resolvers[key] = MultilangRegexURLResolver(r'^/', urlconf)
    return _resolvers[key]


def reverse_for_language(viewname, lang, urlconf=None, args=None, kwargs=None, prefix=None):
    # Copied from Django 1.0 code in django.core.urlresolvers.RegexURLResolver.reverse
    args = args or []
    kwargs = kwargs or {}
    if prefix is None:
        prefix = get_script_prefix()
    resolver = get_resolver(urlconf, lang)

    if args and kwargs:
        raise ValueError("Don't mix *args and **kwargs in call to reverse()!")
    try:
        lookup_view = get_callable(viewname, True)
    except (ImportError, AttributeError), e:
        raise NoReverseMatch("Error importing '%s': %s." % (lookup_view, e))
    if hasattr(resolver, 'get_reverse_dict'):
        possibilities = resolver.get_reverse_dict(lang).getlist(lookup_view)
    else:
        possibilities = resolver.reverse_dict.getlist(lookup_view)
    for possibility, pattern in possibilities:
        for result, params in possibility:
            if args:
                if len(args) != len(params):
                    continue
                unicode_args = [force_unicode(val) for val in args]
                candidate =  result % dict(zip(params, unicode_args))
            else:
                if set(kwargs.keys()) != set(params):
                    continue
                unicode_kwargs = dict([(k, force_unicode(v)) for (k, v) in kwargs.items()])
                candidate = result % unicode_kwargs
            if re.search(u'^%s' % pattern, candidate, re.UNICODE):
                iri = u'%s%s' % (prefix, candidate)
                # If we have a separate domain for lang, put that in the iri
                domain = LANGUAGE_DOMAINS.get(lang, None)
                if domain:
                    iri = u'http://%s%s' % (domain[0], iri)
                return iri_to_uri(iri)
    raise NoReverseMatch("Reverse for '%s' with arguments '%s' and keyword "
            "arguments '%s' not found." % (lookup_view, args, kwargs))


class MultilangRegexURLPattern(RegexURLPattern):
    def __init__(self, regex, callback, default_args=None, name=None):
        # Copied from django.core.urlresolvers.RegexURLPattern, with one change:
        # The regex here is stored as a string instead of as a compiled re object.
        # This allows the code to use the gettext system to translate the URL
        # pattern at resolve time.
        self._raw_regex = regex

        if callable(callback):
            self._callback = callback
        else:
            self._callback = None
            self._callback_str = callback
        self.default_args = default_args or {}
        self.name = name
        self._regex_dict = {}

    def get_regex(self, lang=None):
        lang = lang or get_language()
        return self._regex_dict.setdefault(lang, re.compile(translation(lang).ugettext(self._raw_regex), re.UNICODE))
    regex = property(get_regex)


class MultilangRegexURLResolver(RegexURLResolver):
    def __init__(self, regex, urlconf_name, default_kwargs=None):
        # regex is a string representing a regular expression.
        # urlconf_name is a string representing the module containing urlconfs.
        self._raw_regex = regex
        self.urlconf_name = urlconf_name
        self.callback = None
        self.default_kwargs = default_kwargs or {}
        self._lang_reverse_dicts = {}
        self._regex_dict = {}

    def get_regex(self, lang=None):
        lang = lang or get_language()
        return self._regex_dict.setdefault(lang, re.compile(translation(lang).ugettext(self._raw_regex), re.UNICODE))
    regex = property(get_regex)

    def _build_reverse_dict_for_lang(self, lang):
        reverse_dict = MultiValueDict()
        if hasattr(self.urlconf_module, 'urlpatterns'):
            for pattern in reversed(self.urlconf_module.urlpatterns):
                if hasattr(pattern, 'get_regex'):
                    p_pattern = pattern.get_regex(lang).pattern
                else:
                    p_pattern = pattern.regex.pattern
                if p_pattern.startswith('^'):
                    p_pattern = p_pattern[1:]
                if isinstance(pattern, RegexURLResolver):
                    if hasattr(pattern, 'get_regex'):
                        parent = normalize(pattern.get_regex(lang).pattern)
                    else:
                        parent = normalize(pattern.regex.pattern)
                    if hasattr(pattern, 'get_reverse_dict'):
                        sub_reverse_dict = pattern.get_reverse_dict(lang)
                    else:
                        sub_reverse_dict = pattern.reverse_dict
                    for name in sub_reverse_dict:
                        for matches, pat in sub_reverse_dict.getlist(name):
                            new_matches = []
                            for piece, p_args in parent:
                                new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches])
                            reverse_dict.appendlist(name, (new_matches, p_pattern + pat))
                else:
                    bits = normalize(p_pattern)
                    reverse_dict.appendlist(pattern.callback, (bits, p_pattern))
                    reverse_dict.appendlist(pattern.name, (bits, p_pattern))
        return reverse_dict

    def get_reverse_dict(self, lang=None):
        if lang is None:
            lang = get_language()
        if lang not in self._lang_reverse_dicts:
            self._lang_reverse_dicts[lang] = self._build_reverse_dict_for_lang(lang)
        return self._lang_reverse_dicts[lang]
    reverse_dict = property(get_reverse_dict)


class LangSelectionRegexURLResolver(MultilangRegexURLResolver):
    def __init__(self, urlconf_name, default_kwargs=None):
        # urlconf_name is a string representing the module containing urlconfs.
        self.urlconf_name = urlconf_name
        self.callback = None
        self.default_kwargs = default_kwargs or {}
        self._lang_reverse_dicts = {}
        self._regex_dict = {}

    def get_regex(self, lang=None):
        lang = lang or get_language()
        return re.compile('^%s/' % lang)
    regex = property(get_regex)<commit_msg>Updated url resolver code to support changes made in Django 1.1.<commit_after>import re

from django.conf import settings
from django.core.urlresolvers import RegexURLPattern, RegexURLResolver, get_callable
from django.core.urlresolvers import NoReverseMatch
from django.core.urlresolvers import get_script_prefix
from django.utils.datastructures import MultiValueDict
from django.utils.encoding import iri_to_uri, force_unicode
from django.utils.regex_helper import normalize
from django.utils.translation import get_language
from django.utils.translation.trans_real import translation

from multilang.settings import LANGUAGE_DOMAINS


def turl(regex, view, kwargs=None, name=None, prefix=''):
    # Copied from django.conf.urls.defaults.url
    if isinstance(view, (list,tuple)):
        # For include(...) processing.
        urlconf_module, app_name, namespace = view
        return MultilangRegexURLResolver(regex, urlconf_module, kwargs, app_name=app_name, namespace=namespace)
    else:
        if isinstance(view, basestring):
            if not view:
                raise ImproperlyConfigured('Empty URL pattern view name not permitted (for pattern %r)' % regex)
            if prefix:
                view = prefix + '.' + view
        return MultilangRegexURLPattern(regex, view, kwargs, name)


def lang_prefixed(urlconf_name):
    return LangSelectionRegexURLResolver(urlconf_name)


_resolvers = {}
def get_resolver(urlconf, lang):
    if urlconf is None:
        from django.conf import settings
        urlconf = settings.ROOT_URLCONF
    key = (urlconf, lang)
    if key not in _resolvers:
        _resolvers[key] = MultilangRegexURLResolver(r'^/', urlconf)
    return _resolvers[key]


def reverse_for_language(viewname, lang, urlconf=None, args=None, kwargs=None, prefix=None):
    # Copied from Django 1.0 code in django.core.urlresolvers.RegexURLResolver.reverse
    args = args or []
    kwargs = kwargs or {}
    if prefix is None:
        prefix = get_script_prefix()
    resolver = get_resolver(urlconf, lang)

    if args and kwargs:
        raise ValueError("Don't mix *args and **kwargs in call to reverse()!")
    try:
        lookup_view = get_callable(viewname, True)
    except (ImportError, AttributeError), e:
        raise NoReverseMatch("Error importing '%s': %s." % (lookup_view, e))
    if hasattr(resolver, 'get_reverse_dict'):
        possibilities = resolver.get_reverse_dict(lang).getlist(lookup_view)
    else:
        possibilities = resolver.reverse_dict.getlist(lookup_view)
    for possibility, pattern in possibilities:
        for result, params in possibility:
            if args:
                if len(args) != len(params):
                    continue
                unicode_args = [force_unicode(val) for val in args]
                candidate =  result % dict(zip(params, unicode_args))
            else:
                if set(kwargs.keys()) != set(params):
                    continue
                unicode_kwargs = dict([(k, force_unicode(v)) for (k, v) in kwargs.items()])
                candidate = result % unicode_kwargs
            if re.search(u'^%s' % pattern, candidate, re.UNICODE):
                iri = u'%s%s' % (prefix, candidate)
                # If we have a separate domain for lang, put that in the iri
                domain = LANGUAGE_DOMAINS.get(lang, None)
                if domain:
                    iri = u'http://%s%s' % (domain[0], iri)
                return iri_to_uri(iri)
    raise NoReverseMatch("Reverse for '%s' with arguments '%s' and keyword "
            "arguments '%s' not found." % (lookup_view, args, kwargs))


class MultilangRegexURLPattern(RegexURLPattern):
    def __init__(self, regex, callback, default_args=None, name=None):
        # Copied from django.core.urlresolvers.RegexURLPattern, with one change:
        # The regex here is stored as a string instead of as a compiled re object.
        # This allows the code to use the gettext system to translate the URL
        # pattern at resolve time.
        self._raw_regex = regex

        if callable(callback):
            self._callback = callback
        else:
            self._callback = None
            self._callback_str = callback
        self.default_args = default_args or {}
        self.name = name
        self._regex_dict = {}

    def get_regex(self, lang=None):
        lang = lang or get_language()
        return self._regex_dict.setdefault(lang, re.compile(translation(lang).ugettext(self._raw_regex), re.UNICODE))
    regex = property(get_regex)


class MultilangRegexURLResolver(RegexURLResolver):
    def __init__(self, regex, urlconf_name, default_kwargs=None, app_name=None, namespace=None):
        # regex is a string representing a regular expression.
        # urlconf_name is a string representing the module containing urlconfs.
        self._raw_regex = regex
        self.urlconf_name = urlconf_name
        self.callback = None
        self.default_kwargs = default_kwargs or {}
        self.namespace = namespace
        self.app_name = app_name
        self._lang_reverse_dicts = {}
        self._namespace_dict = None
        self._app_dict = None
        self._regex_dict = {}

    def get_regex(self, lang=None):
        lang = lang or get_language()
        return self._regex_dict.setdefault(lang, re.compile(translation(lang).ugettext(self._raw_regex), re.UNICODE))
    regex = property(get_regex)

    def _build_reverse_dict_for_lang(self, lang):
        reverse_dict = MultiValueDict()
        if hasattr(self.urlconf_module, 'urlpatterns'):
            for pattern in reversed(self.urlconf_module.urlpatterns):
                if hasattr(pattern, 'get_regex'):
                    p_pattern = pattern.get_regex(lang).pattern
                else:
                    p_pattern = pattern.regex.pattern
                if p_pattern.startswith('^'):
                    p_pattern = p_pattern[1:]
                if isinstance(pattern, RegexURLResolver):
                    if hasattr(pattern, 'get_regex'):
                        parent = normalize(pattern.get_regex(lang).pattern)
                    else:
                        parent = normalize(pattern.regex.pattern)
                    if hasattr(pattern, 'get_reverse_dict'):
                        sub_reverse_dict = pattern.get_reverse_dict(lang)
                    else:
                        sub_reverse_dict = pattern.reverse_dict
                    for name in sub_reverse_dict:
                        for matches, pat in sub_reverse_dict.getlist(name):
                            new_matches = []
                            for piece, p_args in parent:
                                new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches])
                            reverse_dict.appendlist(name, (new_matches, p_pattern + pat))
                else:
                    bits = normalize(p_pattern)
                    reverse_dict.appendlist(pattern.callback, (bits, p_pattern))
                    reverse_dict.appendlist(pattern.name, (bits, p_pattern))
        return reverse_dict

    def get_reverse_dict(self, lang=None):
        if lang is None:
            lang = get_language()
        if lang not in self._lang_reverse_dicts:
            self._lang_reverse_dicts[lang] = self._build_reverse_dict_for_lang(lang)
        return self._lang_reverse_dicts[lang]
    reverse_dict = property(get_reverse_dict)


class LangSelectionRegexURLResolver(MultilangRegexURLResolver):
    def __init__(self, urlconf_name, default_kwargs=None, app_name=None, namespace=None):
        # urlconf_name is a string representing the module containing urlconfs.
        self.urlconf_name = urlconf_name
        self.callback = None
        self.default_kwargs = default_kwargs or {}
        self.namespace = namespace
        self.app_name = app_name
        self._lang_reverse_dicts = {}
        self._namespace_dict = None
        self._app_dict = None
        self._regex_dict = {}

    def get_regex(self, lang=None):
        lang = lang or get_language()
        return re.compile('^%s/' % lang)
    regex = property(get_regex)
{'fraction_non_alphanumeric': 0.08570342687020967, 'fraction_numerical': 0.0006967758282130867, 'mean_word_length': 2.4062567421790724, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1042739', 'n_tokens_mistral': 4451, 'n_tokens_neox': 4309, 'n_words': 1121}