<commit_before>import datetime
import decimal
import itertools
import re
import time
import urllib2
import uuid
import warnings
from operator import itemgetter

import six

try:
    import dateutil
except ImportError:
    dateutil = None
else:
    import dateutil.parser

import pymongo
import gridfs
from bson import Binary, DBRef, SON, ObjectId
try:
    from bson.int64 import Int64
except ImportError:
    Int64 = long

from mongoengine.errors import ValidationError
from mongoengine.python_support import (PY3, bin_type, txt_type,
                                        str_types, StringIO)
from base import (BaseField, ComplexBaseField, ObjectIdField, GeoJsonBaseField,
                  get_document, BaseDocument)
from queryset import DO_NOTHING, QuerySet
from document import Document, EmbeddedDocument
from connection import get_db, DEFAULT_CONNECTION_NAME

try:
    from PIL import Image, ImageOps
except ImportError:
    Image = None
    ImageOps = None

__all__ = [
    'StringField', 'URLField', 'EmailField', 'IntField', 'LongField',
    'FloatField', 'DecimalField', 'BooleanField', 'DateTimeField',
    'ComplexDateTimeField', 'EmbeddedDocumentField', 'ObjectIdField',
    'GenericEmbeddedDocumentField', 'DynamicField', 'ListField',
    'SortedListField', 'EmbeddedDocumentListField', 'DictField',
    'MapField', 'ReferenceField', 'CachedReferenceField',
    'GenericReferenceField', 'BinaryField', 'GridFSError', 'GridFSProxy',
    'FileField', 'ImageGridFsProxy', 'ImproperlyConfigured', 'ImageField',
    'GeoPointField', 'PointField', 'LineStringField', 'PolygonField',
    'SequenceField', 'UUIDField', 'MultiPointField', 'MultiLineStringField',
    'MultiPolygonField', 'GeoJsonBaseField']

RECURSIVE_REFERENCE_CONSTANT = 'self'


class StringField(BaseField):
    """A unicode string field.
    """

    def __init__(self, regex=None, max_length=None, min_length=None, **kwargs):
        self.regex = re.compile(regex) if regex else None
        self.max_length = max_length
        self.min_length = min_length
        super(StringField, self).__init__(**kwargs)

    def to_python(self, value):
        if isinstance(value, unicode):
            return value
        try:
            value = value.decode('utf-8')
        except Exception:
            pass
        return value

    def validate(self, value):
        if not isinstance(value, basestring):
            self.error('StringField only accepts string values')

        if self.max_length is not None and len(value) > self.max_length:
            self.error('String value is too long')

        if self.min_length is not None and len(value) < self.min_length:
            self.error('String value is too short')

        if self.regex is not None and self.regex.match(value) is None:
            self.error('String value did not match validation regex')

    def lookup_member(self, member_name):
        return None

    def prepare_query_value(self, op, value):
        if not isinstance(op, basestring):
            return value

        if op.lstrip('i') in ('startswith', 'endswith', 'contains', 'exact'):
            flags = 0
            if op.startswith('i'):
                flags = re.IGNORECASE
                op = op.lstrip('i')

            regex = r'%s'
            if op == 'startswith':
                regex = r'^%s'
            elif op == 'endswith':
                regex = r'%s$'
            elif op == 'exact':
                regex = r'^%s$'

            # escape unsafe characters which could lead to a re.error
            value = re.escape(value)
            value = re.compile(regex % value, flags)
        return super(StringField, self).prepare_query_value(op, value)


class URLField(StringField):
    """A field that validates input as an URL.

    .. versionadded:: 0.3
    """

    _URL_REGEX = re.compile(
        r'^(?:[a-z0-9\.\-]*)://'  # scheme is validated separately
        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}(?<!-)\.?)|'  # domain...
        r'localhost|'  # localhost...
        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|'  # ...or ipv4
        r'\[?[A-F0-9]*:[A-F0-9:]+\]?)'  # ...or ipv6
        r'(?::\d+)?'  # optional port
        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
    _URL_SCHEMES = ['http', 'https', 'ftp', 'ftps']

    def __init__(self, verify_exists=False, url_regex=None, schemes=None, **kwargs):
        self.verify_exists = verify_exists
        self.url_regex = url_regex or self._URL_REGEX
        self.schemes = schemes or self._URL_SCHEMES
        super(URLField, self).__init__(**kwargs)

    def validate(self, value):
        # Check first if the scheme is valid
        scheme = value.split('://')[0].lower()
        if scheme not in self.schemes:
            self.error('Invalid scheme {} in URL: {}'.format(scheme, value))
            return

        # Then check full URL
        if not self.url_regex.match(value):
            self.error('Invalid URL: {}'.format(value))
            return

        if self.verify_exists:
            warnings.warn(
                "The URLField verify_exists argument has intractable security "
                "and performance issues. Accordingly, it has been deprecated.",
                DeprecationWarning)
            try:
                request = urllib2.Request(value)
                urllib2.urlopen(request)
            except Exception, e:
                self.error('This URL appears to be a broken link: %s' % e)


class EmailField(StringField):
    """A field that validates input as an E-Mail-Address.

    .. versionadded:: 0.4
    """

    EMAIL_REGEX = re.compile(
        # dot-atom
        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"
        # quoted-string
        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'
        # domain (max length of an ICAAN TLD is 22 characters)
        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}|[A-Z0-9-]{2,}(?<!-))$', re.IGNORECASE
    )

    def validate(self, value):
        if not EmailField.EMAIL_REGEX.match(value):
            self.error('Invalid Mail-address: %s' % value)
        super(EmailField, self).validate(value)


class IntField(BaseField):<commit_msg>Use English spelling for "email address"<commit_after>import datetime
import decimal
import itertools
import re
import time
import urllib2
import uuid
import warnings
from operator import itemgetter

import six

try:
    import dateutil
except ImportError:
    dateutil = None
else:
    import dateutil.parser

import pymongo
import gridfs
from bson import Binary, DBRef, SON, ObjectId
try:
    from bson.int64 import Int64
except ImportError:
    Int64 = long

from mongoengine.errors import ValidationError
from mongoengine.python_support import (PY3, bin_type, txt_type,
                                        str_types, StringIO)
from base import (BaseField, ComplexBaseField, ObjectIdField, GeoJsonBaseField,
                  get_document, BaseDocument)
from queryset import DO_NOTHING, QuerySet
from document import Document, EmbeddedDocument
from connection import get_db, DEFAULT_CONNECTION_NAME

try:
    from PIL import Image, ImageOps
except ImportError:
    Image = None
    ImageOps = None

__all__ = [
    'StringField', 'URLField', 'EmailField', 'IntField', 'LongField',
    'FloatField', 'DecimalField', 'BooleanField', 'DateTimeField',
    'ComplexDateTimeField', 'EmbeddedDocumentField', 'ObjectIdField',
    'GenericEmbeddedDocumentField', 'DynamicField', 'ListField',
    'SortedListField', 'EmbeddedDocumentListField', 'DictField',
    'MapField', 'ReferenceField', 'CachedReferenceField',
    'GenericReferenceField', 'BinaryField', 'GridFSError', 'GridFSProxy',
    'FileField', 'ImageGridFsProxy', 'ImproperlyConfigured', 'ImageField',
    'GeoPointField', 'PointField', 'LineStringField', 'PolygonField',
    'SequenceField', 'UUIDField', 'MultiPointField', 'MultiLineStringField',
    'MultiPolygonField', 'GeoJsonBaseField']

RECURSIVE_REFERENCE_CONSTANT = 'self'


class StringField(BaseField):
    """A unicode string field.
    """

    def __init__(self, regex=None, max_length=None, min_length=None, **kwargs):
        self.regex = re.compile(regex) if regex else None
        self.max_length = max_length
        self.min_length = min_length
        super(StringField, self).__init__(**kwargs)

    def to_python(self, value):
        if isinstance(value, unicode):
            return value
        try:
            value = value.decode('utf-8')
        except Exception:
            pass
        return value

    def validate(self, value):
        if not isinstance(value, basestring):
            self.error('StringField only accepts string values')

        if self.max_length is not None and len(value) > self.max_length:
            self.error('String value is too long')

        if self.min_length is not None and len(value) < self.min_length:
            self.error('String value is too short')

        if self.regex is not None and self.regex.match(value) is None:
            self.error('String value did not match validation regex')

    def lookup_member(self, member_name):
        return None

    def prepare_query_value(self, op, value):
        if not isinstance(op, basestring):
            return value

        if op.lstrip('i') in ('startswith', 'endswith', 'contains', 'exact'):
            flags = 0
            if op.startswith('i'):
                flags = re.IGNORECASE
                op = op.lstrip('i')

            regex = r'%s'
            if op == 'startswith':
                regex = r'^%s'
            elif op == 'endswith':
                regex = r'%s$'
            elif op == 'exact':
                regex = r'^%s$'

            # escape unsafe characters which could lead to a re.error
            value = re.escape(value)
            value = re.compile(regex % value, flags)
        return super(StringField, self).prepare_query_value(op, value)


class URLField(StringField):
    """A field that validates input as an URL.

    .. versionadded:: 0.3
    """

    _URL_REGEX = re.compile(
        r'^(?:[a-z0-9\.\-]*)://'  # scheme is validated separately
        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}(?<!-)\.?)|'  # domain...
        r'localhost|'  # localhost...
        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|'  # ...or ipv4
        r'\[?[A-F0-9]*:[A-F0-9:]+\]?)'  # ...or ipv6
        r'(?::\d+)?'  # optional port
        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
    _URL_SCHEMES = ['http', 'https', 'ftp', 'ftps']

    def __init__(self, verify_exists=False, url_regex=None, schemes=None, **kwargs):
        self.verify_exists = verify_exists
        self.url_regex = url_regex or self._URL_REGEX
        self.schemes = schemes or self._URL_SCHEMES
        super(URLField, self).__init__(**kwargs)

    def validate(self, value):
        # Check first if the scheme is valid
        scheme = value.split('://')[0].lower()
        if scheme not in self.schemes:
            self.error('Invalid scheme {} in URL: {}'.format(scheme, value))
            return

        # Then check full URL
        if not self.url_regex.match(value):
            self.error('Invalid URL: {}'.format(value))
            return

        if self.verify_exists:
            warnings.warn(
                "The URLField verify_exists argument has intractable security "
                "and performance issues. Accordingly, it has been deprecated.",
                DeprecationWarning)
            try:
                request = urllib2.Request(value)
                urllib2.urlopen(request)
            except Exception, e:
                self.error('This URL appears to be a broken link: %s' % e)


class EmailField(StringField):
    """A field that validates input as an email address.

    .. versionadded:: 0.4
    """

    EMAIL_REGEX = re.compile(
        # dot-atom
        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"
        # quoted-string
        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"'
        # domain (max length of an ICAAN TLD is 22 characters)
        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}|[A-Z0-9-]{2,}(?<!-))$', re.IGNORECASE
    )

    def validate(self, value):
        if not EmailField.EMAIL_REGEX.match(value):
            self.error('Invalid email address: %s' % value)
        super(EmailField, self).validate(value)


class IntField(BaseField):
{'fraction_non_alphanumeric': 0.14352111547399632, 'fraction_numerical': 0.016988540748457407, 'mean_word_length': 2.5244281276475573, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6052984', 'n_tokens_mistral': 4091, 'n_tokens_neox': 3887, 'n_words': 954}