<commit_before># coding=utf-8
from __future__ import unicode_literals
import six
import warnings
from inspect import getargspec
from django.utils.importlib import import_module

try:
    from django.template.context import BaseContext
except ImportError:
    from django.template.context import Context as BaseContext  # Django < 1.2
try:
    from django.utils.encoding import smart_text, smart_bytes
except ImportError:
    from django.utils.encoding import smart_unicode as smart_text, smart_str as smart_bytes

from .cache import get_cache
from .conf import settings
from .const import DEFAULT_TIMEOUT
from .loaders import render


def import_by_path(path):
    m, _, f = settings.VIEWLET_ARGS_DIGEST_FUNCTION.rpartition('.')
    return getattr(import_module(m), f)


arg_digest_func = import_by_path(settings.VIEWLET_ARGS_DIGEST_FUNCTION)


class Viewlet(object):
    """
    Representation of a viewlet
    """

    def __init__(self, library, name=None, template=None, key=None, timeout=DEFAULT_TIMEOUT, using=None,
                 cached=True):
        self.library = library
        self.name = name
        self.template = template
        self.key = key
        self.has_args = False
        self.cache_alias = using
        self.cache = get_cache(alias=using)
        if timeout is None:
            # Handle infinite caching, due to Django's cache backend not respecting 0
            self.timeout = settings.VIEWLET_INFINITE_CACHE_TIMEOUT
        elif timeout is DEFAULT_TIMEOUT:
            self.timeout = self.cache.default_timeout
        else:
            self.timeout = timeout
        if not cached:
            self.timeout = 0
            warnings.warn('Keyword argument "cached" is deprecated, use timeout=0 to disable cache',
                          DeprecationWarning)

        self.arg_digest_func = arg_digest_func

    def register(self, func):
        """
        Initial decorator wrapper that configures the viewlet instance,
        adds it to the library and then returns a pointer to the call
        function as the actual wrapper
        """
        self.viewlet_func = func
        self.viewlet_func_args = getargspec(func).args
        self.has_args = len(self.viewlet_func_args) > 1

        if not self.name:
            self.name = getattr(func, 'func_name', getattr(func, '__name__'))

        key = u'viewlet:' + self.name
        if self.is_using_cache():
            if self.key and self.has_args:
                assert '{dig}' in self.key, \
                    u"If you want to use your custom key for a viewlet which has arguments, please add " \
                    u"`{dig}` to the key where digest of the arguments will be inserted."

            if not self.key:
                self.key = key
                if self.has_args:
                    self.key += u':{dig}'
        else:
            self.key = key

        self.library.add(self)

        def call_with_refresh(*args, **kwargs):
            return self.call(*args, **kwargs)
        setattr(call_with_refresh, 'refresh', self.refresh)

        return call_with_refresh

    def _build_args(self, *args, **kwargs):
        viewlet_func_kwargs = dict((self.viewlet_func_args[i], args[i]) for i in range(0, len(args)))
        viewlet_func_kwargs.update(dict((k, kwargs[k]) for k in kwargs if k in self.viewlet_func_args))
        return [viewlet_func_kwargs.get(arg) for arg in self.viewlet_func_args]

    def _build_cache_key(self, *args):
        """
        Build cache key based on viewlet argument except initial context argument.
        """
        if self.has_args:
            key = self.key.format(dig=self.arg_digest_func(self, args))
        else:
            key = self.key
        max_len = settings.VIEWLET_MAX_CACHE_KEY_LENGTH
        assert len(key) <= max_len, \
            u"Viewlet cache key is too long: len(`{key}`) > {max_len}".format(key=key, max_len=max_len)
        return key

    def _cache_get(self, key):
        return self.cache.get(key)<commit_msg>Restore unicode after cache get<commit_after># coding=utf-8
from __future__ import unicode_literals
import six
import warnings
from inspect import getargspec
from django.utils.importlib import import_module

try:
    from django.template.context import BaseContext
except ImportError:
    from django.template.context import Context as BaseContext  # Django < 1.2
try:
    from django.utils.encoding import smart_text, smart_bytes
except ImportError:
    from django.utils.encoding import smart_unicode as smart_text, smart_str as smart_bytes

from .cache import get_cache
from .conf import settings
from .const import DEFAULT_TIMEOUT
from .loaders import render


def import_by_path(path):
    m, _, f = settings.VIEWLET_ARGS_DIGEST_FUNCTION.rpartition('.')
    return getattr(import_module(m), f)


arg_digest_func = import_by_path(settings.VIEWLET_ARGS_DIGEST_FUNCTION)


class Viewlet(object):
    """
    Representation of a viewlet
    """

    def __init__(self, library, name=None, template=None, key=None, timeout=DEFAULT_TIMEOUT, using=None,
                 cached=True):
        self.library = library
        self.name = name
        self.template = template
        self.key = key
        self.has_args = False
        self.cache_alias = using
        self.cache = get_cache(alias=using)
        if timeout is None:
            # Handle infinite caching, due to Django's cache backend not respecting 0
            self.timeout = settings.VIEWLET_INFINITE_CACHE_TIMEOUT
        elif timeout is DEFAULT_TIMEOUT:
            self.timeout = self.cache.default_timeout
        else:
            self.timeout = timeout
        if not cached:
            self.timeout = 0
            warnings.warn('Keyword argument "cached" is deprecated, use timeout=0 to disable cache',
                          DeprecationWarning)

        self.arg_digest_func = arg_digest_func

    def register(self, func):
        """
        Initial decorator wrapper that configures the viewlet instance,
        adds it to the library and then returns a pointer to the call
        function as the actual wrapper
        """
        self.viewlet_func = func
        self.viewlet_func_args = getargspec(func).args
        self.has_args = len(self.viewlet_func_args) > 1

        if not self.name:
            self.name = getattr(func, 'func_name', getattr(func, '__name__'))

        key = u'viewlet:' + self.name
        if self.is_using_cache():
            if self.key and self.has_args:
                assert '{dig}' in self.key, \
                    u"If you want to use your custom key for a viewlet which has arguments, please add " \
                    u"`{dig}` to the key where digest of the arguments will be inserted."

            if not self.key:
                self.key = key
                if self.has_args:
                    self.key += u':{dig}'
        else:
            self.key = key

        self.library.add(self)

        def call_with_refresh(*args, **kwargs):
            return self.call(*args, **kwargs)
        setattr(call_with_refresh, 'refresh', self.refresh)

        return call_with_refresh

    def _build_args(self, *args, **kwargs):
        viewlet_func_kwargs = dict((self.viewlet_func_args[i], args[i]) for i in range(0, len(args)))
        viewlet_func_kwargs.update(dict((k, kwargs[k]) for k in kwargs if k in self.viewlet_func_args))
        return [viewlet_func_kwargs.get(arg) for arg in self.viewlet_func_args]

    def _build_cache_key(self, *args):
        """
        Build cache key based on viewlet argument except initial context argument.
        """
        if self.has_args:
            key = self.key.format(dig=self.arg_digest_func(self, args))
        else:
            key = self.key
        max_len = settings.VIEWLET_MAX_CACHE_KEY_LENGTH
        assert len(key) <= max_len, \
            u"Viewlet cache key is too long: len(`{key}`) > {max_len}".format(key=key, max_len=max_len)
        return key

    def _cache_get(self, key):
        s = self.cache.get(key)
        if isinstance(s, six.binary_type):
            s = smart_text(s)
        return s
{'fraction_non_alphanumeric': 0.08428976922127607, 'fraction_numerical': 0.0019745773170430705, 'mean_word_length': 2.4179671024884013, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4288085', 'n_tokens_mistral': 2351, 'n_tokens_neox': 2292, 'n_words': 668}