<commit_before>""" Cross-object data auditing

Schema validation allows for checking values within a single object.
We also need to perform higher order checking between linked objects.
"""

import logging
import venusian
from .embedding import embed

logger = logging.getLogger(__name__)


def includeme(config):
    config.registry['auditor'] = Auditor()
    config.add_directive('add_audit_checker', add_audit_checker)
    config.add_request_method(audit, 'audit')


# Same as logging
_levelNames = {
    0: 'NOTSET',
    10: 'DEBUG',
    20: 'INFO',
    30: 'DCC_ACTION',
    40: 'WARNING',
    50: 'STANDARDS_FAILURE',
    60: 'ERROR',
    'DEBUG': 10,
    'ERROR': 60,
    'INFO': 20,
    'NOTSET': 0,
    'WARNING': 40,
    'STANDARDS_FAILURE': 50,
    'DCC_ACTION': 30,
}


class AuditFailure(Exception):
    def __init__(self, category, detail=None, level=0):
        super(AuditFailure, self)
        self.category = category
        self.detail = detail
        if not isinstance(level, int):
            level = _levelNames[level]
        self.level = level
        self.path = None

    def __json__(self, request=None):
        return {
            'category': self.category,
            'detail': self.detail,
            'level': self.level,
            'level_name': _levelNames[self.level],
            'path': self.path,
        }


class Auditor(object):
    """ Data audit manager
    """
    _order = 0

    def __init__(self):
        self.type_checkers = {}

    def add_audit_checker(self, checker, item_type, condition=None, frame='embedded'):
        checkers = self.type_checkers.setdefault(item_type, [])
        self._order += 1  # consistent execution ordering
        if not isinstance(frame, basestring):
            frame = tuple(sorted(frame))
        checkers.append((self._order, checker, condition, frame))

    def audit(self, request, types, path, **kw):
        if isinstance(types, basestring):
            types = [types]
        checkers = set()
        checkers.update(*(self.type_checkers.get(item_type, ()) for item_type in types))
        errors = []
        system = {}
        system.update(kw)
        for order, checker, condition, frame in sorted(checkers):
            if frame is None:
                uri = path
            elif isinstance(frame, basestring):
                uri = '%s?frame%s' % (path, frame)
            else:
                uri = '%s?frame=expand&expand=%s' % (path, '&expand='.join(frame))
            value = embed(request, uri)

            if condition is not None:
                try:
                    if not condition(value, system):
                        continue
                except Exception as e:
                    detail = '%s: %r' % (checker.__name__, e)
                    errors.append(AuditFailure('audit condition error', detail, 'ERROR'))
                    logger.warning('audit condition error auditing %s', path, exc_info=True)
                    continue
            try:
                try:
                    result = checker(value, system)
                except AuditFailure as e:
                    e.path = path
                    errors.append(e)
                    continue
                if result is None:
                    continue
                if isinstance(result, AuditFailure):
                    result = [result]
                for item in result:
                    if isinstance(item, AuditFailure):
                        item.path = path
                        errors.append(item)<commit_msg>Make sure audit errors are copyable<commit_after>""" Cross-object data auditing

Schema validation allows for checking values within a single object.
We also need to perform higher order checking between linked objects.
"""

import logging
import venusian
from .embedding import embed

logger = logging.getLogger(__name__)


def includeme(config):
    config.registry['auditor'] = Auditor()
    config.add_directive('add_audit_checker', add_audit_checker)
    config.add_request_method(audit, 'audit')


# Same as logging
_levelNames = {
    0: 'NOTSET',
    10: 'DEBUG',
    20: 'INFO',
    30: 'DCC_ACTION',
    40: 'WARNING',
    50: 'STANDARDS_FAILURE',
    60: 'ERROR',
    'DEBUG': 10,
    'ERROR': 60,
    'INFO': 20,
    'NOTSET': 0,
    'WARNING': 40,
    'STANDARDS_FAILURE': 50,
    'DCC_ACTION': 30,
}


class AuditFailure(Exception):
    def __init__(self, category, detail=None, level=0):
        super(AuditFailure, self)
        self.category = category
        self.detail = detail
        if not isinstance(level, int):
            level = _levelNames[level]
        self.level = level

    def __json__(self, request=None):
        return {
            'category': self.category,
            'detail': self.detail,
            'level': self.level,
            'level_name': _levelNames[self.level],
        }


class Auditor(object):
    """ Data audit manager
    """
    _order = 0

    def __init__(self):
        self.type_checkers = {}

    def add_audit_checker(self, checker, item_type, condition=None, frame='embedded'):
        checkers = self.type_checkers.setdefault(item_type, [])
        self._order += 1  # consistent execution ordering
        if not isinstance(frame, basestring):
            frame = tuple(sorted(frame))
        checkers.append((self._order, checker, condition, frame))

    def audit(self, request, types, path, **kw):
        if isinstance(types, basestring):
            types = [types]
        checkers = set()
        checkers.update(*(self.type_checkers.get(item_type, ()) for item_type in types))
        errors = []
        system = {}
        system.update(kw)
        for order, checker, condition, frame in sorted(checkers):
            if frame is None:
                uri = path
            elif isinstance(frame, basestring):
                uri = '%s?frame%s' % (path, frame)
            else:
                uri = '%s?frame=expand&expand=%s' % (path, '&expand='.join(frame))
            value = embed(request, uri)

            if condition is not None:
                try:
                    if not condition(value, system):
                        continue
                except Exception as e:
                    detail = '%s: %r' % (checker.__name__, e)
                    errors.append(AuditFailure('audit condition error', detail, 'ERROR'))
                    logger.warning('audit condition error auditing %s', path, exc_info=True)
                    continue
            try:
                try:
                    result = checker(value, system)
                except AuditFailure as e:
                    e = e.__json__(request)
                    e['path'] = path
                    errors.append(e)
                    continue
                if result is None:
                    continue
                if isinstance(result, AuditFailure):
                    result = [result]
                for item in result:
                    if isinstance(item, AuditFailure):
                        item = item.__json__(request)
                        item['path'] = path
                        errors.append(item)
{'fraction_non_alphanumeric': 0.10640380313199105, 'fraction_numerical': 0.008109619686800894, 'mean_word_length': 1.8194718171068192, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6307427', 'n_tokens_mistral': 2033, 'n_tokens_neox': 1919, 'n_words': 524}