<commit_before>import base64
try:
    import cPickle as pickle
except ImportError:
    import pickle
import logging
from pprint import pformat

from django.conf import settings
from django.template import (Template, Context, TemplateDoesNotExist,
    TemplateSyntaxError)
from django.utils.encoding import smart_unicode
from django.utils.html import escape
from django.views.debug import ExceptionReporter, linebreak_iter

from indexer.models import Index

class ImprovedExceptionReporter(ExceptionReporter):
    def __init__(self, request, exc_type, exc_value, frames, template_info=None):
        ExceptionReporter.__init__(self, request, exc_type, exc_value, None)
        self.frames = frames
        self._template = template_info

    def get_traceback_frames(self):
        return self.frames

    def get_template_exception_info(self):
        template_source, start, end, name = self._template
        context_lines = 10
        line = 0
        upto = 0
        source_lines = []
        before = during = after = ""
        for num, next in enumerate(linebreak_iter(template_source)):
            if start >= upto and end <= next:
                line = num
                before = escape(template_source[upto:start])
                during = escape(template_source[start:end])
                after = escape(template_source[end:next])
            source_lines.append( (num, escape(template_source[upto:next])) )
            upto = next
        total = len(source_lines)

        top = max(1, line - context_lines)
        bottom = min(total, line + 1 + context_lines)

        self.template_info = {
            'message': self.exc_value.args[0],
            'source_lines': source_lines[top:bottom],
            'before': before,
            'during': during,
            'after': after,
            'top': top,
            'bottom': bottom,
            'total': total,
            'line': line,
            'name': name,
        }

    def get_traceback_html(self):
        "Return HTML code for traceback."

        if issubclass(self.exc_type, TemplateDoesNotExist):
            self.template_does_not_exist = True
        if self._template:
            self.get_template_exception_info()

        frames = self.get_traceback_frames()

        unicode_hint = ''
        if issubclass(self.exc_type, UnicodeError):
            start = getattr(self.exc_value, 'start', None)
            end = getattr(self.exc_value, 'end', None)
            if start is not None and end is not None:
                unicode_str = self.exc_value.args[1]
                unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
        t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
        c = Context({
            'exception_type': self.exc_type.__name__,
            'exception_value': smart_unicode(self.exc_value, errors='replace'),
            'unicode_hint': unicode_hint,
            'frames': frames,
            'lastframe': frames[-1],
            'request': self.request,
            'template_info': self.template_info,
            'template_does_not_exist': self.template_does_not_exist,
        })
        return t.render(c)

class FakeRequest(object):
    GET = {}
    POST = {}
    META = {}
    COOKIES = {}
    FILES = {}
    
    def __repr__(self):
        # Since this is called as part of error handling, we need to be very
        # robust against potentially malformed input.
        try:
            get = pformat(self.GET)
        except:
            get = '<could not parse>'
        try:
            post = pformat(self.POST)
        except:
            post = '<could not parse>'
        try:
            cookies = pformat(self.COOKIES)
        except:
            cookies = '<could not parse>'
        try:
            meta = pformat(self.META)
        except:
            meta = '<could not parse>'
        return '<Request\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % \
            (get, post, cookies, meta)

    def build_absolute_uri(self): return self.url

TECHNICAL_500_TEMPLATE = """
<div id="summary">
  <h1>{{ exception_type }}{% if request.path_info %} at {{ request.path_info|escape }}{% endif %}</h1>
  <pre class="exception_value">{{ exception_value|escape }}</pre>
  <table class="meta">
    <tr>
      <th>Request Method:</th>
      <td>{{ request.META.REQUEST_METHOD }}</td>
    </tr>
    <tr>
      <th>Request URL:</th>
      <td>{{ request.build_absolute_uri|escape }}</td>
    </tr>
    <tr>
      <th>Exception Type:</th>
      <td>{{ exception_type }}</td>
    </tr>
    <tr>
      <th>Exception Value:</th>
      <td><pre>{{ exception_value|escape }}</pre></td>
    </tr>
    <tr>
      <th>Exception Location:</th>
      <td>{{ lastframe.filename|escape }} in {{ lastframe.function|escape }}, line {{ lastframe.lineno }}</td><commit_msg>Remove URL from base exception title. Link URL in request information.<commit_after>import base64
try:
    import cPickle as pickle
except ImportError:
    import pickle
import logging
from pprint import pformat

from django.conf import settings
from django.template import (Template, Context, TemplateDoesNotExist,
    TemplateSyntaxError)
from django.utils.encoding import smart_unicode
from django.utils.html import escape
from django.views.debug import ExceptionReporter, linebreak_iter

from indexer.models import Index

class ImprovedExceptionReporter(ExceptionReporter):
    def __init__(self, request, exc_type, exc_value, frames, template_info=None):
        ExceptionReporter.__init__(self, request, exc_type, exc_value, None)
        self.frames = frames
        self._template = template_info

    def get_traceback_frames(self):
        return self.frames

    def get_template_exception_info(self):
        template_source, start, end, name = self._template
        context_lines = 10
        line = 0
        upto = 0
        source_lines = []
        before = during = after = ""
        for num, next in enumerate(linebreak_iter(template_source)):
            if start >= upto and end <= next:
                line = num
                before = escape(template_source[upto:start])
                during = escape(template_source[start:end])
                after = escape(template_source[end:next])
            source_lines.append( (num, escape(template_source[upto:next])) )
            upto = next
        total = len(source_lines)

        top = max(1, line - context_lines)
        bottom = min(total, line + 1 + context_lines)

        self.template_info = {
            'message': self.exc_value.args[0],
            'source_lines': source_lines[top:bottom],
            'before': before,
            'during': during,
            'after': after,
            'top': top,
            'bottom': bottom,
            'total': total,
            'line': line,
            'name': name,
        }

    def get_traceback_html(self):
        "Return HTML code for traceback."

        if issubclass(self.exc_type, TemplateDoesNotExist):
            self.template_does_not_exist = True
        if self._template:
            self.get_template_exception_info()

        frames = self.get_traceback_frames()

        unicode_hint = ''
        if issubclass(self.exc_type, UnicodeError):
            start = getattr(self.exc_value, 'start', None)
            end = getattr(self.exc_value, 'end', None)
            if start is not None and end is not None:
                unicode_str = self.exc_value.args[1]
                unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
        t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
        c = Context({
            'exception_type': self.exc_type.__name__,
            'exception_value': smart_unicode(self.exc_value, errors='replace'),
            'unicode_hint': unicode_hint,
            'frames': frames,
            'lastframe': frames[-1],
            'request': self.request,
            'template_info': self.template_info,
            'template_does_not_exist': self.template_does_not_exist,
        })
        return t.render(c)

class FakeRequest(object):
    GET = {}
    POST = {}
    META = {}
    COOKIES = {}
    FILES = {}
    
    def __repr__(self):
        # Since this is called as part of error handling, we need to be very
        # robust against potentially malformed input.
        try:
            get = pformat(self.GET)
        except:
            get = '<could not parse>'
        try:
            post = pformat(self.POST)
        except:
            post = '<could not parse>'
        try:
            cookies = pformat(self.COOKIES)
        except:
            cookies = '<could not parse>'
        try:
            meta = pformat(self.META)
        except:
            meta = '<could not parse>'
        return '<Request\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % \
            (get, post, cookies, meta)

    def build_absolute_uri(self): return self.url

TECHNICAL_500_TEMPLATE = """
<div id="summary">
  <h1>{{ exception_type }}</h1>
  <pre class="exception_value">{{ exception_value|escape }}</pre>
  <table class="meta">
    <tr>
      <th>Request Method:</th>
      <td>{{ request.META.REQUEST_METHOD }}</td>
    </tr>
    {% if request.path_info %}
    <tr>
      <th>Request URL:</th>
      <td><a href="{{ request.build_absolute_uri|escape }}">{{ request.build_absolute_uri|escape }}</a></td>
    </tr>
    {% endif %}
    <tr>
      <th>Exception Type:</th>
      <td>{{ exception_type }}</td>
    </tr>
    <tr>
      <th>Exception Value:</th>
      <td><pre>{{ exception_value|escape }}</pre></td>
    </tr>
    <tr>
      <th>Exception Location:</th>
      <td>{{ lastframe.filename|escape }} in {{ lastframe.function|escape }}, line {{ lastframe.lineno }}</td>
{'fraction_non_alphanumeric': 0.11926325429937926, 'fraction_numerical': 0.005088022794342118, 'mean_word_length': 2.3102054563826204, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '763780', 'n_tokens_mistral': 2960, 'n_tokens_neox': 2853, 'n_words': 685}