<commit_before>from django.conf import settings
from django.utils.html import strip_tags
from django.utils.text import unescape_entities
from django.utils.encoding import force_unicode

ROA_DJANGO_ERRORS = getattr(settings, 'ROA_DJANGO_ERRORS', False)

class ROAException(Exception):
    def __init__(self, exception):
        if ROA_DJANGO_ERRORS:
            self.msg = exception.message
            self.status_code = exception.status_code
        else:
            self.msg = force_unicode(exception)

    def __str__(self):
        return ROA_DJANGO_ERRORS and self.parse_django_error() or self.msg
    
    def parse_django_error(self):
        """Extract the summary part of a Django HTML error."""
        summary = self.msg.split('<body>\n<div id="summary">\n  ', 1)[1]\
                          .split('<th>Python Executable:</th>', 1)[0]
        result = []
        title = None
        for line in strip_tags(summary).split('\n'):
            line_content = unescape_entities(line.strip())
            if line_content:
                if line_content.endswith(':'):
                    title = line_content
                elif title is None:
                    title = "%s:" % line_content
                else:<commit_msg>Do not parse Django's error messages if this is not an HTML file (useful for Bad requests and so on)<commit_after>from django.conf import settings
from django.utils.html import strip_tags
from django.utils.text import unescape_entities
from django.utils.encoding import force_unicode

ROA_DJANGO_ERRORS = getattr(settings, 'ROA_DJANGO_ERRORS', False)

class ROAException(Exception):
    def __init__(self, exception):
        if ROA_DJANGO_ERRORS:
            self.msg = exception.message
            self.status_code = exception.status_code
        else:
            self.msg = force_unicode(exception)

    def __str__(self):
        if ROA_DJANGO_ERRORS and '<body>' in self.msg:
            return self.parse_django_error()
        return self.msg
    
    def parse_django_error(self):
        """Extract the summary part of a Django HTML error."""
        summary = self.msg.split('<body>\n<div id="summary">\n  ', 1)[1]\
                          .split('<th>Python Executable:</th>', 1)[0]
        result = []
        title = None
        for line in strip_tags(summary).split('\n'):
            line_content = unescape_entities(line.strip())
            if line_content:
                if line_content.endswith(':'):
                    title = line_content
                elif title is None:
                    title = "%s:" % line_content
                else:
{'fraction_non_alphanumeric': 0.09762633996937213, 'fraction_numerical': 0.0030627871362940277, 'mean_word_length': 2.2459627329192546, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2944806', 'n_tokens_mistral': 753, 'n_tokens_neox': 725, 'n_words': 188}