<commit_before>import logging
import sys
import urllib
import urllib2
import uuid

import django
from django.conf import settings
from django.utils.encoding import force_unicode
from django.utils.hashcompat import md5_constructor

from sentry import conf

_FILTER_CACHE = None
def get_filters():
    global _FILTER_CACHE
    
    if _FILTER_CACHE is None:
        
        filters = []
        for filter_ in conf.FILTERS:
            module_name, class_name = filter_.rsplit('.', 1)
            try:
                module = __import__(module_name, {}, {}, class_name)
                filter_ = getattr(module, class_name)
            except Exception:
                logging.exception('Unable to import %s' % (filter_,))
                continue
            filters.append(filter_)
        _FILTER_CACHE = filters
    for f in _FILTER_CACHE:
        yield f

def get_db_engine(alias='default'):
    has_multidb = django.VERSION >= (1, 2)
    if has_multidb:
        value = settings.DATABASES[alias]['ENGINE']
    else:
        assert alias == 'default', 'You cannot fetch a database engine other than the default on Django < 1.2'
        value = settings.DATABASE_ENGINE
    return value.rsplit('.', 1)[-1]

def construct_checksum(level=logging.ERROR, class_name='', traceback='', message='', **kwargs):
    checksum = md5_constructor(str(level))
    checksum.update(class_name or '')
    if traceback:
        traceback = '\n'.join(traceback.split('\n')[:-3])
    message = traceback or message
    if isinstance(message, unicode):
        message = message.encode('utf-8', 'replace')
    checksum.update(message)
    return checksum.hexdigest()

def varmap(func, var):
    if isinstance(var, dict):
        return dict((k, varmap(func, v)) for k, v in var.iteritems())
    elif isinstance(var, (list, tuple)):
        return [varmap(func, f) for f in var]
    else:
        return func(var)

def transform(value):
    # TODO: make this extendable
    # TODO: include some sane defaults, like UUID
    # TODO: dont coerce strings to unicode, leave them as strings
    if isinstance(value, (tuple, list, set, frozenset)):
        return type(value)(transform(o) for o in value)
    elif isinstance(value, uuid.UUID):
        return repr(value)
    elif isinstance(value, dict):
        return dict((k, transform(v)) for k, v in value.iteritems())
    elif isinstance(value, unicode):
        return to_unicode(value)
    elif isinstance(value, str):
        try:
            return str(value)
        except:
            return to_unicode(value)
    elif hasattr(value, '__sentry__'):
        return value.__sentry__()
    elif not isinstance(value, (int, bool)) and value is not None:
        # XXX: we could do transform(repr(value)) here
        return to_unicode(value)
    return value

def to_unicode(value):
    try:
        value = force_unicode(value)
    except (UnicodeEncodeError, UnicodeDecodeError):
        value = '(Error decoding value)'
    except Exception: # in some cases we get a different exception<commit_msg>Ensure we force values to unicode that Django may not (fixes GH-69)<commit_after>import logging
import sys
import urllib
import urllib2
import uuid

import django
from django.conf import settings
from django.utils.encoding import force_unicode
from django.utils.hashcompat import md5_constructor

from sentry import conf

_FILTER_CACHE = None
def get_filters():
    global _FILTER_CACHE
    
    if _FILTER_CACHE is None:
        
        filters = []
        for filter_ in conf.FILTERS:
            module_name, class_name = filter_.rsplit('.', 1)
            try:
                module = __import__(module_name, {}, {}, class_name)
                filter_ = getattr(module, class_name)
            except Exception:
                logging.exception('Unable to import %s' % (filter_,))
                continue
            filters.append(filter_)
        _FILTER_CACHE = filters
    for f in _FILTER_CACHE:
        yield f

def get_db_engine(alias='default'):
    has_multidb = django.VERSION >= (1, 2)
    if has_multidb:
        value = settings.DATABASES[alias]['ENGINE']
    else:
        assert alias == 'default', 'You cannot fetch a database engine other than the default on Django < 1.2'
        value = settings.DATABASE_ENGINE
    return value.rsplit('.', 1)[-1]

def construct_checksum(level=logging.ERROR, class_name='', traceback='', message='', **kwargs):
    checksum = md5_constructor(str(level))
    checksum.update(class_name or '')
    if traceback:
        traceback = '\n'.join(traceback.split('\n')[:-3])
    message = traceback or message
    if isinstance(message, unicode):
        message = message.encode('utf-8', 'replace')
    checksum.update(message)
    return checksum.hexdigest()

def varmap(func, var):
    if isinstance(var, dict):
        return dict((k, varmap(func, v)) for k, v in var.iteritems())
    elif isinstance(var, (list, tuple)):
        return [varmap(func, f) for f in var]
    else:
        return func(var)

def transform(value):
    # TODO: make this extendable
    # TODO: include some sane defaults, like UUID
    # TODO: dont coerce strings to unicode, leave them as strings
    if isinstance(value, (tuple, list, set, frozenset)):
        return type(value)(transform(o) for o in value)
    elif isinstance(value, uuid.UUID):
        return repr(value)
    elif isinstance(value, dict):
        return dict((k, transform(v)) for k, v in value.iteritems())
    elif isinstance(value, unicode):
        return to_unicode(value)
    elif isinstance(value, str):
        try:
            return str(value)
        except:
            return to_unicode(value)
    elif hasattr(value, '__sentry__'):
        return value.__sentry__()
    elif not isinstance(value, (int, bool)) and value is not None:
        # XXX: we could do transform(repr(value)) here
        return to_unicode(value)
    return value

def to_unicode(value):
    try:
        value = unicode(force_unicode(value))
    except (UnicodeEncodeError, UnicodeDecodeError):
        value = '(Error decoding value)'
    except Exception: # in some cases we get a different exception
{'fraction_non_alphanumeric': 0.10249307479224377, 'fraction_numerical': 0.004236597686165879, 'mean_word_length': 2.824299065420561, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1022202', 'n_tokens_mistral': 1799, 'n_tokens_neox': 1753, 'n_words': 554}