<commit_before>#!/usr/bin/env python

from __future__ import print_function

import sys
import os
import os.path as path
from collections import defaultdict

import xml.sax
from xml.sax.handler import ContentHandler, ErrorHandler

class Error(object):

    def __init__(self, severity, line, column, message):
        self.severity = severity
        self.line = line
        self.column = column
        self.message = message

class CheckStyleHandler(ContentHandler):

    def __init__(self):
        self.errors = defaultdict(list)
        self.currentFile = None

    def startElement(self, name, attrs):
        if name == "file":
            self.currentFile = attrs.get("name")
        elif name == "error":
            self.handleError(attrs)

    def handleError(self, attrs):
        err = Error(
                attrs.get('severity'),
                attrs.get('line'),
                attrs.get('column'),
                attrs.get('message'))
        self.errors[self.currentFile].append(err)

class NervousErrorHandler(object):

    def error(self, exception):
        raise exception

    def warning(self, exception):
        raise exception

ERROR_FMT = '  line: {0.line:>4}, column: {0.column:>3} - {0.severity} - {0.message}'

report = sys.argv[1]

error_count = 0

print("Parsing", report)

with open(report) as f:
    handler = CheckStyleHandler()
    xml.sax.parse(f, handler, NervousErrorHandler())
    for file_name, errors in handler.errors.items():
        print("FILE:", file_name)
        for e in errors:
            print(ERROR_FMT.format(e))
            error_count += 1

sys.exit(error_count)

<commit_msg>Ignore issues to do with compiled classes.<commit_after>#!/usr/bin/env python

from __future__ import print_function

import sys
import os
import os.path as path
from collections import defaultdict

import xml.sax
from xml.sax.handler import ContentHandler, ErrorHandler

class Error(object):

    def __init__(self, severity, line, column, message):
        self.severity = severity
        self.line = line
        self.column = column
        self.message = message

class CheckStyleHandler(ContentHandler):

    def __init__(self):
        self.errors = defaultdict(list)
        self.currentFile = None

    def startElement(self, name, attrs):
        if name == "file":
            self.currentFile = attrs.get("name")
        elif name == "error":
            self.handleError(attrs)

    def can_ignore(self, err):
        return 'Unable to get class information' in err.message:

    def handleError(self, attrs):
        err = Error(
                attrs.get('severity'),
                attrs.get('line'),
                attrs.get('column'),
                attrs.get('message'))
        if not self.can_ignore(err):
            self.errors[self.currentFile].append(err)

class NervousErrorHandler(object):

    def error(self, exception):
        raise exception

    def warning(self, exception):
        raise exception

ERROR_FMT = '  line: {0.line:>4}, column: {0.column:>3} - {0.severity} - {0.message}'

report = sys.argv[1]

error_count = 0

with open(report) as f:
    handler = CheckStyleHandler()
    xml.sax.parse(f, handler, NervousErrorHandler())
    for file_name, errors in handler.errors.items():
        print("FILE:", file_name)
        for e in errors:
            print(ERROR_FMT.format(e))
            error_count += 1

sys.exit(error_count)


{'fraction_non_alphanumeric': 0.11179397131987123, 'fraction_numerical': 0.0052677787532923615, 'mean_word_length': 2.575313807531381, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5612729', 'n_tokens_mistral': 1021, 'n_tokens_neox': 992, 'n_words': 260}