<commit_before>from __future__ import print_function

import logging
import pprint
from collections import defaultdict

from six import string_types

from .schema import SCHEMA, mk_sample_event

"""
Data Pipeline validation functions
"""

SCHEMA_DICT = None
REQ_FIELDS = None
CHECKS = {'req': 'Fields "{}" are missing, but required. \n{} present',
          'size': 'Field "{}" is too large (size limit {})',
          'type': 'Field "{}" should be {}',
          'not_in_schema': 'Field "{}" not in schema. {}',}

log = logging.getLogger(__name__)

def _create_schema_dict():
    global SCHEMA_DICT, REQ_FIELDS

    SCHEMA_DICT = defaultdict(dict)
    for field_conditions in SCHEMA:
        conditions = field_conditions
        if conditions['type'] == object:
            conditions['type'] = dict
        if conditions['type'] == str:
            conditions['type'] = string_types

        SCHEMA_DICT[field_conditions['key']] = conditions

    # all required and top-level fields should be present.
    REQ_FIELDS = set([k for k, v in SCHEMA_DICT.items() if v.get('req') or not v.get('available_with_field')])
    SCHEMA_DICT = dict(SCHEMA_DICT)
_create_schema_dict()


def _handle_warning(check_type, field, value, cond, raise_error=True):
    """If raise, raise an error. Otherwise just log."""
    msg = "Validation Error:  " + CHECKS[check_type].format(field, cond)
    if raise_error:
        raise ValueError(msg, value, type(value))
    else:
        log.warn(msg, value, type(value))

    return False
<commit_msg>Add a custom error class so it can be caught specially<commit_after>from __future__ import print_function

import logging
import pprint
from collections import defaultdict

from six import string_types

from .schema import SCHEMA, mk_sample_event

"""
Data Pipeline validation functions
"""

SCHEMA_DICT = None
REQ_FIELDS = None
CHECKS = {'req': 'Fields "{}" are missing, but required. \n{} present',
          'size': 'Field "{}" is too large (size limit {})',
          'type': 'Field "{}" should be {}',
          'not_in_schema': 'Field "{}" not in schema. {}',}

log = logging.getLogger(__name__)


class SchemaValidationError(Exception):
    """An event fails the validation test."""
    pass


def _create_schema_dict():
    global SCHEMA_DICT, REQ_FIELDS

    SCHEMA_DICT = defaultdict(dict)
    for field_conditions in SCHEMA:
        conditions = field_conditions
        if conditions['type'] == object:
            conditions['type'] = dict
        if conditions['type'] == str:
            conditions['type'] = string_types

        SCHEMA_DICT[field_conditions['key']] = conditions

    # all required and top-level fields should be present.
    REQ_FIELDS = set([k for k, v in SCHEMA_DICT.items() if v.get('req') or not v.get('available_with_field')])
    SCHEMA_DICT = dict(SCHEMA_DICT)
_create_schema_dict()


def _handle_warning(check_type, field, value, cond, raise_error=True):
    """If raise, raise an error. Otherwise just log."""
    msg = "Validation Error:  " + CHECKS[check_type].format(field, cond)
    if raise_error:
        raise SchemaValidationError(msg, value, type(value))
    else:
        log.warn(msg, value, type(value))

    return False

{'fraction_non_alphanumeric': 0.11790393013100436, 'fraction_numerical': 0.0, 'mean_word_length': 3.5683760683760686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '748221', 'n_tokens_mistral': 996, 'n_tokens_neox': 975, 'n_words': 289}