<commit_before>from __future__ import print_function
import sys
from collections import defaultdict
from itertools import islice
from logging import Filter
import traceback
from datetime import timedelta, datetime

from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter

from celery.utils.mail import ErrorMail
from dimagi.utils.django.email import send_HTML_email as _send_HTML_email
from django.core import mail
from django.http import HttpRequest
from django.utils.log import AdminEmailHandler
from django.views.debug import SafeExceptionReporterFilter, get_exception_reporter_filter
from django.template.loader import render_to_string
from corehq.apps.analytics.utils import analytics_enabled_for_email
from corehq.util.view_utils import get_request
from corehq.util.datadog.utils import get_url_group, sanitize_url
from corehq.util.datadog.gauges import datadog_counter
from corehq.util.datadog.metrics import ERROR_COUNT
from corehq.util.datadog.const import DATADOG_UNKNOWN


def clean_exception(exception):
    """
    Takes an Exception instance and strips potentially sensitive information
    """
    from django.conf import settings
    if settings.DEBUG:
        return exception

    # couchdbkit doesn't provide a better way for us to catch this exception
    if (
        isinstance(exception, AssertionError) and
        exception.message.startswith('received an invalid response of type')
    ):
        message = ("It looks like couch returned an invalid response to "
                   "couchdbkit.  This could contain sensitive information, "
                   "so it's being redacted.")
        return exception.__class__(message)

    return exception


def get_sanitized_request_repr(request):
    """
    Sanitizes sensitive data inside request object, if request has been marked sensitive
    via Django decorator, django.views.decorators.debug.sensitive_post_parameters
    """
    if isinstance(request, HttpRequest):
        filter = get_exception_reporter_filter(request)
        return repr(filter.get_post_parameters(request))

    return request


class HqAdminEmailHandler(AdminEmailHandler):
    """
    Custom AdminEmailHandler to include additional details which can be supplied as follows:

    logger.error(message,
        extra={
            'details': {'domain': 'demo', 'user': 'user1'}
        }
    )
    """

    def get_context(self, record):
        try:
            request = record.request
        except Exception:
            request = None

        request_repr = get_sanitized_request_repr(request)

        tb_list = []
        code = None
        if record.exc_info:
            etype, _value, tb = record.exc_info
            value = clean_exception(_value)
            tb_list = ['Traceback (most recent call first):\n']
            formatted_exception = traceback.format_exception_only(etype, value)
            tb_list.extend(formatted_exception)
            extracted_tb = list(reversed(traceback.extract_tb(tb)))
            code = self.get_code(extracted_tb)
            tb_list.extend(traceback.format_list(extracted_tb))
            stack_trace = '\n'.join(tb_list)
            subject = '%s: %s' % (record.levelname,<commit_msg>no ciruclar import<commit_after>from __future__ import print_function
import sys
from collections import defaultdict
from itertools import islice
from logging import Filter
import traceback
from datetime import timedelta, datetime

from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter

from celery.utils.mail import ErrorMail
from dimagi.utils.django.email import send_HTML_email as _send_HTML_email
from django.core import mail
from django.http import HttpRequest
from django.utils.log import AdminEmailHandler
from django.views.debug import SafeExceptionReporterFilter, get_exception_reporter_filter
from django.template.loader import render_to_string
from corehq.apps.analytics.utils import analytics_enabled_for_email
from corehq.util.view_utils import get_request
from corehq.util.datadog.utils import get_url_group, sanitize_url
from corehq.util.datadog.metrics import ERROR_COUNT
from corehq.util.datadog.const import DATADOG_UNKNOWN


def clean_exception(exception):
    """
    Takes an Exception instance and strips potentially sensitive information
    """
    from django.conf import settings
    if settings.DEBUG:
        return exception

    # couchdbkit doesn't provide a better way for us to catch this exception
    if (
        isinstance(exception, AssertionError) and
        exception.message.startswith('received an invalid response of type')
    ):
        message = ("It looks like couch returned an invalid response to "
                   "couchdbkit.  This could contain sensitive information, "
                   "so it's being redacted.")
        return exception.__class__(message)

    return exception


def get_sanitized_request_repr(request):
    """
    Sanitizes sensitive data inside request object, if request has been marked sensitive
    via Django decorator, django.views.decorators.debug.sensitive_post_parameters
    """
    if isinstance(request, HttpRequest):
        filter = get_exception_reporter_filter(request)
        return repr(filter.get_post_parameters(request))

    return request


class HqAdminEmailHandler(AdminEmailHandler):
    """
    Custom AdminEmailHandler to include additional details which can be supplied as follows:

    logger.error(message,
        extra={
            'details': {'domain': 'demo', 'user': 'user1'}
        }
    )
    """

    def get_context(self, record):
        from corehq.util.datadog.gauges import datadog_counter
        try:
            request = record.request
        except Exception:
            request = None

        request_repr = get_sanitized_request_repr(request)

        tb_list = []
        code = None
        if record.exc_info:
            etype, _value, tb = record.exc_info
            value = clean_exception(_value)
            tb_list = ['Traceback (most recent call first):\n']
            formatted_exception = traceback.format_exception_only(etype, value)
            tb_list.extend(formatted_exception)
            extracted_tb = list(reversed(traceback.extract_tb(tb)))
            code = self.get_code(extracted_tb)
            tb_list.extend(traceback.format_list(extracted_tb))
            stack_trace = '\n'.join(tb_list)
            subject = '%s: %s' % (record.levelname,
{'fraction_non_alphanumeric': 0.06647266043804564, 'fraction_numerical': 0.00030632562413845916, 'mean_word_length': 3.3159286186384667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2179063', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1723, 'n_words': 501}