<commit_before>from voluptuous import MultipleInvalid, Schema
from wtforms.validators import ValidationError

from datatypes.exceptions import *


class Validator(object):
    def __init__(self, required=False, extra=True):
        self.schema = Schema(schema=self.define_schema(), required=required, extra=extra)

    def clean_input(self, data):
        return data

    def define_schema(self):
        raise NoSchemaException()

    def to_canonical_form(self, data):
        return data

    def validate(self):
        raise Exception("You must define a validate method")


class DictionaryValidator(Validator):
    def __init__(self):
        super(DictionaryValidator, self).__init__()
        self.error_dictionary = self.define_error_dictionary()

    def clean_input(self, dictionary):
        filtered = {}

        for k, v in dictionary.iteritems():
            if isinstance(v, dict):
                filtered[k] = self.clean_input(v)
            else:
                if v is not None:
                    filtered[k] = v

        return filtered

    def validate(self, data):
        def translate_voluptuous_errors(voluptuous_exception):
            def flatten_error(error_path):
                return {error_path: self.error_dictionary.get(error_path, e.error_message)
                        for e in voluptuous_exception.errors if e.path}

            def flatten_path(exception):
                return '.'.join(map(lambda x: str(x), filter(lambda x: not str(x).isdigit(), exception.path)))

            flattened_errors = {}
            map(lambda e:
                flattened_errors.update(flatten_error(flatten_path(e))),
                voluptuous_exception.errors)
            return flattened_errors

        try:
            self.schema(self.clean_input(data))
        except MultipleInvalid as exception:
            raise DataDoesNotMatchSchemaException(cause=exception,
                                                  value=data,
                                                  field_errors=translate_voluptuous_errors(exception))

    def define_error_dictionary(self):
        raise NoErrorDictionaryDefined()


class SingleValueValidator(Validator):
    class WtfDatatypeValidator(object):
        def __init__(self, validator, message):
            self.validator = validator
            self.message = message
<commit_msg>Slight cleanup of error code.<commit_after>from voluptuous import MultipleInvalid, Schema
from wtforms.validators import ValidationError

from datatypes.exceptions import *


class Validator(object):
    def __init__(self, required=False, extra=True):
        self.schema = Schema(schema=self.define_schema(), required=required, extra=extra)

    def clean_input(self, data):
        return data

    def define_schema(self):
        raise NoSchemaException()

    def to_canonical_form(self, data):
        return data

    def validate(self, data):
        raise Exception("You must define a validate method")


class DictionaryValidator(Validator):
    def __init__(self):
        super(DictionaryValidator, self).__init__()
        self.error_dictionary = self.define_error_dictionary()

    def clean_input(self, dictionary):
        filtered = {}

        for k, v in dictionary.iteritems():
            if isinstance(v, dict):
                filtered[k] = self.clean_input(v)
            else:
                if v is not None:
                    filtered[k] = v

        return filtered

    def validate(self, data):
        try:
            self.schema(self.clean_input(data))
        except MultipleInvalid as exception:
            def translate_voluptuous_errors(voluptuous_exception):
                def flatten_error(error_path):
                    return {error_path: self.error_dictionary.get(error_path, e.error_message)
                            for e in voluptuous_exception.errors if e.path}

                def flatten_path(error):
                    return '.'.join(map(lambda x: str(x), filter(lambda x: not str(x).isdigit(), error.path)))

                flattened_errors = {}
                map(lambda e: flattened_errors.update(flatten_error(flatten_path(e))), voluptuous_exception.errors)
                return flattened_errors

            raise DataDoesNotMatchSchemaException(cause=exception,
                                                  value=data,
                                                  field_errors=translate_voluptuous_errors(exception))

    def define_error_dictionary(self):
        raise NoErrorDictionaryDefined()


class SingleValueValidator(Validator):
    class WtfDatatypeValidator(object):
        def __init__(self, validator, message):
            self.validator = validator
            self.message = message

{'fraction_non_alphanumeric': 0.0808421052631579, 'fraction_numerical': 0.0, 'mean_word_length': 2.2563399588759423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6227474', 'n_tokens_mistral': 1245, 'n_tokens_neox': 1174, 'n_words': 281}