<commit_before>from datetime import datetime, date, time
from pyutils.encoding import force_str, force_unicode
from time import strptime
import re

class Invalid(Exception):
    def __init__(self, path=(), messages=(), children=()):
        if not isinstance(messages, (list, tuple)):
            messages = [messages]
        self.messages = {}
        if messages:
            self.messages[path] = list(messages)
        self.add(children)

    def __repr__(self):
        return str(self)

    def __str__(self):
        return force_str(unicode(self))

    def __unicode__(self):
        result = []
        for path, messages in self.flattened().items():
            prefix = path + ': ' if path else ''
            for index, message in enumerate(messages):
                if index == 0:
                    result.append(prefix + message)
                else:
                    result.append(message.rjust(len(prefix)))

        if len(result) == 1:
            return result[0]
        return '\n' + '\n'.join(result)

    def flattened(self):
        return {'.'.join(map(force_unicode, path)): submessages
                for path, submessages in self.messages.items()}

    def add(self, errors):
        if not hasattr(errors, '__iter__'):
            errors = (errors,)

        for error in errors:
            for path, messages in error.messages.items():
                self.messages.setdefault(path, []).extend(messages)

class MaxLength(object):
    def __init__(self, max_length):
        self.max_length = max_length

    def check(self, value, path):
        if len(value) > self.max_length:
            raise Invalid(path, 'Ensure this value has at most %d characters '
                                '(it has %d).' % (self.max_length, len(value)))

class MinValue(object):
    def __init__(self, min_value):
        self.min_value = min_value

    def check(self, value, path):
        if value < self.min_value:
            raise Invalid(path, 'This value must be larger than %s.' % self.min_value)

class MaxValue(object):
    def __init__(self, max_value):
        self.max_value = max_value

    def check(self, value, path):
        if value < self.min_value:
            raise Invalid(path, 'This value must be smaller than %s.' % self.max_value)

class Equals(object):
    def __init__(self, value):
        self.value = value

    def check(self, value, path):
        if value != self.value:
            raise Invalid(path, 'This value must be equal to %r.' % self.value)

class In(object):
    def __init__(self, choice):
        self.choice = choice

    def check(self, value, path):
        if value not in self.choice:
            raise Invalid(path, 'This value must be one of: %s'
                                % ', '.join(map(repr, self.choice)))

email_re = re.compile(
    # dot-atom
    r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"
    # quoted-string, see also http://tools.ietf.org/html/rfc2822#section-3.2.5
    r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"'
    # domain
    r')@((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$)'
    # literal form, ipv4 address (SMTP 4.1.3)
    r'|\[(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\]$',
    re.IGNORECASE)

class EmailValidator(object):
    def check(self, value, path):
        if not email_re.match(value):
            # Trivial case failed. Try for possible IDN domain-part
            if value and u'@' in value:
                parts = value.split(u'@')
                try:
                    parts[-1] = parts[-1].encode('idna')
                except UnicodeError:
                    raise
                value = u'@'.join(parts)
            if email_re.match(value):
                return
            raise Invalid(path, 'Enter a valid e-mail address.')

whitespace_re = re.compile(r'\s+')
spam_re = re.compile('|'.join((
    r'we can increase ((your|the) )?ranking',
    r'our experts? in (Search Engine Optimization|SEO)',
    r'our (Search Engine Optimization|SEO) expert',
    r'we offer (quality )?(Search Engine Optimization|SEO)',
    r'the best (Search Engine Optimization|SEO) service',
    r'our dedicated team of (Search Engine Optimization|SEO) Professionals',
    r'I have found the best kept secret',
    r'and I want to share it with you',
    )), re.IGNORECASE)

class Spam(object):
    def __init__(self, should_check=lambda value: True):
        self.should_check = should_check

    def check(self, value, path):
        if self.should_check(value) and spam_re.search(whitespace_re.sub(' ', value)):
            raise Invalid(path, "Sorry, your message looks like spam.")

class Schema(object):
    default_validators = []

    def __init__(self, null=False, optional=False, validators=None):
        self.null = null
        self.optional = optional
        self.validators = self.default_validators[:]
        if validators:
            self.validators.extend(validators)

    def convert(self, value, path=()):<commit_msg>workaround for pyjs syntax error<commit_after>from datetime import datetime, date, time
from pyutils.encoding import force_str, force_unicode
from time import strptime
import re

class Invalid(Exception):
    def __init__(self, path=(), messages=(), children=()):
        if not isinstance(messages, (list, tuple)):
            messages = [messages]
        self.messages = {}
        if messages:
            self.messages[path] = list(messages)
        self.add(children)

    def __repr__(self):
        return str(self)

    def __str__(self):
        return force_str(unicode(self))

    def __unicode__(self):
        result = []
        for path, messages in self.flattened().items():
            prefix = path + ': ' if path else ''
            for index, message in enumerate(messages):
                if index == 0:
                    result.append(prefix + message)
                else:
                    result.append(message.rjust(len(prefix)))

        if len(result) == 1:
            return result[0]
        return '\n' + '\n'.join(result)

    def flattened(self):
        return {'.'.join(map(force_unicode, path)): submessages
                for path, submessages in self.messages.items()}

    def add(self, errors):
        if not hasattr(errors, '__iter__'):
            errors = (errors,)

        for error in errors:
            for path, messages in error.messages.items():
                self.messages.setdefault(path, []).extend(messages)

class MaxLength(object):
    def __init__(self, max_length):
        self.max_length = max_length

    def check(self, value, path):
        if len(value) > self.max_length:
            raise Invalid(path, 'Ensure this value has at most %d characters '
                                '(it has %d).' % (self.max_length, len(value)))

class MinValue(object):
    def __init__(self, min_value):
        self.min_value = min_value

    def check(self, value, path):
        if value < self.min_value:
            raise Invalid(path, 'This value must be larger than %s.' % self.min_value)

class MaxValue(object):
    def __init__(self, max_value):
        self.max_value = max_value

    def check(self, value, path):
        if value < self.min_value:
            raise Invalid(path, 'This value must be smaller than %s.' % self.max_value)

class Equals(object):
    def __init__(self, value):
        self.value = value

    def check(self, value, path):
        if value != self.value:
            raise Invalid(path, 'This value must be equal to %r.' % self.value)

class In(object):
    def __init__(self, choice):
        self.choice = choice

    def check(self, value, path):
        if value not in self.choice:
            raise Invalid(path, 'This value must be one of: %s'
                                % ', '.join(map(repr, self.choice)))

email_re = re.compile(
    # dot-atom
    r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"
    # quoted-string, see also http://tools.ietf.org/html/rfc2822#section-3.2.5
    r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"'
    # domain
    r')@((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$)'
    # literal form, ipv4 address (SMTP 4.1.3)
    r'|\[(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\]$',
    re.IGNORECASE)

class EmailValidator(object):
    def check(self, value, path):
        if not email_re.match(value):
            # Trivial case failed. Try for possible IDN domain-part
            if value and u'@' in value:
                parts = value.split(u'@')
                try:
                    parts[-1] = parts[-1].encode('idna')
                except UnicodeError:
                    raise
                value = u'@'.join(parts)
            if email_re.match(value):
                return
            raise Invalid(path, 'Enter a valid e-mail address.')

whitespace_re = re.compile(r'\s+')
spam_re = re.compile('|'.join((
    r'we can increase ((your|the) )?ranking',
    r'our experts? in (Search Engine Optimization|SEO)',
    r'our (Search Engine Optimization|SEO) expert',
    r'we offer (quality )?(Search Engine Optimization|SEO)',
    r'the best (Search Engine Optimization|SEO) service',
    r'our dedicated team of (Search Engine Optimization|SEO) Professionals',
    r'I have found the best kept secret',
    r'and I want to share it with you',
    )), re.IGNORECASE)

def should_check_spam(value):
    return True

class Spam(object):
    def __init__(self, should_check=should_check_spam):
        self.should_check = should_check

    def check(self, value, path):
        if self.should_check(value) and spam_re.search(whitespace_re.sub(' ', value)):
            raise Invalid(path, "Sorry, your message looks like spam.")

class Schema(object):
    default_validators = []

    def __init__(self, null=False, optional=False, validators=None):
        self.null = null
        self.optional = optional
        self.validators = self.default_validators[:]
        if validators:
            self.validators.extend(validators)

    def convert(self, value, path=()):
{'fraction_non_alphanumeric': 0.14635585718501032, 'fraction_numerical': 0.017507622700895054, 'mean_word_length': 2.318537859007833, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '297907', 'n_tokens_mistral': 3339, 'n_tokens_neox': 3182, 'n_words': 809}