<commit_before>import json
import re
from dateutil import parser as dateutil_parser
from configdb import exceptions
from configdb.db import acl
from configdb.db import validation

ENTITY_NAME_PATTERN = re.compile(r'^[-_a-z0-9]+$')
FIELD_NAME_PATTERN = re.compile(r'^[a-z][_a-z0-9]*$')
DEFAULT_ACL = {'r': '*', 'w': '*'}


class Field(acl.AclMixin, validation.ValidatorMixin):
    """A generic field of our schema.

    Complex fields that aren't directly supported by the network
    transport encoding (JSON) can inherit from this class and
    implement serialization using the to_net() / from_net() methods.

    When implementing serialization, take care of passing through the
    None value unchanged, as it has a special meaning (missing value).
    Also, if serialization fails, try to raise ValueError.
    """

    def __init__(self, entity, name, attrs):
        self.set_acl(attrs.pop('acl', None))
        self.set_validator(attrs.pop('validator', None))
        self.type = attrs.pop('type', 'string')
        self.name = name
        self.attrs = attrs

    def is_relation(self):
        return False

    def to_net(self, value):
        """Convert value for this field to net format."""
        return value

    from_net = to_net


class BoolField(Field):

    def __init__(self, entity, name, attrs):
        # Force a specific validator.
        attrs['validator'] = 'bool'
        Field.__init__(self, entity, name, attrs)


class DateTimeField(Field):

    def to_net(self, value):
        if value:
            return value.isoformat()

    def from_net(self, value):
        if value:
            return dateutil_parser.parse(value)


class Relation(Field):
    """A relation to another class.

    All relations are many-to-many.
    """

    def __init__(self, entity, name, attrs):
        self.local_name = entity.name
        self.remote_name = attrs.pop('rel')
        attrs['validator'] = 'relation'
        Field.__init__(self, entity, name, attrs)

    def is_relation(self):
        return True

    def to_net(self, value):
        # Return list-of-strings as they are, but assume list of
        # database objects otherwise.
        if (isinstance(value, list)
            and (not len(value) or not isinstance(value[0], basestring))):
            return [x.name for x in value]
        else:
            return value


# Table of known field types.
TYPE_MAP = {
    'string': Field,
    'password': Field,
    'text': Field,
    'binary': Field,
    'int': Field,
    'bool': BoolField,
    'datetime': DateTimeField,
    'relation': Relation,
    }

KNOWN_TYPES = TYPE_MAP.keys()


class Entity(acl.AclMixin):
    """A schema entity ('table', or 'class', equivalently)."""

    def __init__(self, table_name, table_def):
        self.name = table_name
        self.description = None
        self.fields = {}
        for name, attrs in table_def.iteritems():
            # Check some special entity attributes.
            if name == '_acl':
                self.set_acl(attrs)
            elif name == '_help':
                self.description = attrs
            elif FIELD_NAME_PATTERN.match(name):
                # Set some defaults for the special 'name' field.
                if name == 'name':
                    attrs.update({'unique': True,
                                  'index': True,
                                  'nullable': False})
                ftype = attrs.get('type', 'string').lower()
                if ftype not in TYPE_MAP:
                    raise exceptions.SchemaError(
                        'field "%s" has unknown type "%s"' % (name, ftype))
                self.fields[name] = TYPE_MAP[ftype](self, name, attrs)
            else:
                raise exceptions.SchemaError(
                    'invalid field name "%s"' % name)
        if 'name' not in self.fields:
            raise exceptions.SchemaError('missing required "name" field')

    def to_net(self, item, ignore_missing=False):
        if isinstance(item, dict):
            attr_getter = lambda x, y: x.get(y)
        else:<commit_msg>the "id" field is reserved<commit_after>import json
import re
from dateutil import parser as dateutil_parser
from configdb import exceptions
from configdb.db import acl
from configdb.db import validation

ENTITY_NAME_PATTERN = re.compile(r'^[-_a-z0-9]+$')
FIELD_NAME_PATTERN = re.compile(r'^[a-z][_a-z0-9]*$')
DEFAULT_ACL = {'r': '*', 'w': '*'}


class Field(acl.AclMixin, validation.ValidatorMixin):
    """A generic field of our schema.

    Complex fields that aren't directly supported by the network
    transport encoding (JSON) can inherit from this class and
    implement serialization using the to_net() / from_net() methods.

    When implementing serialization, take care of passing through the
    None value unchanged, as it has a special meaning (missing value).
    Also, if serialization fails, try to raise ValueError.
    """

    def __init__(self, entity, name, attrs):
        self.set_acl(attrs.pop('acl', None))
        self.set_validator(attrs.pop('validator', None))
        self.type = attrs.pop('type', 'string')
        self.name = name
        self.attrs = attrs

    def is_relation(self):
        return False

    def to_net(self, value):
        """Convert value for this field to net format."""
        return value

    from_net = to_net


class BoolField(Field):

    def __init__(self, entity, name, attrs):
        # Force a specific validator.
        attrs['validator'] = 'bool'
        Field.__init__(self, entity, name, attrs)


class DateTimeField(Field):

    def to_net(self, value):
        if value:
            return value.isoformat()

    def from_net(self, value):
        if value:
            return dateutil_parser.parse(value)


class Relation(Field):
    """A relation to another class.

    All relations are many-to-many.
    """

    def __init__(self, entity, name, attrs):
        self.local_name = entity.name
        self.remote_name = attrs.pop('rel')
        attrs['validator'] = 'relation'
        Field.__init__(self, entity, name, attrs)

    def is_relation(self):
        return True

    def to_net(self, value):
        # Return list-of-strings as they are, but assume list of
        # database objects otherwise.
        if (isinstance(value, list)
            and (not len(value) or not isinstance(value[0], basestring))):
            return [x.name for x in value]
        else:
            return value


# Table of known field types.
TYPE_MAP = {
    'string': Field,
    'password': Field,
    'text': Field,
    'binary': Field,
    'int': Field,
    'bool': BoolField,
    'datetime': DateTimeField,
    'relation': Relation,
    }

KNOWN_TYPES = TYPE_MAP.keys()


class Entity(acl.AclMixin):
    """A schema entity ('table', or 'class', equivalently)."""

    def __init__(self, table_name, table_def):
        self.name = table_name
        self.description = None
        self.fields = {}
        for name, attrs in table_def.iteritems():
            # Check some special entity attributes.
            if name == '_acl':
                self.set_acl(attrs)
            elif name == '_help':
                self.description = attrs
            elif name == 'id':
                raise exceptions.SchemaError(
                    'the "id" field is reserved')
            elif FIELD_NAME_PATTERN.match(name):
                # Set some defaults for the special 'name' field.
                if name == 'name':
                    attrs.update({'unique': True,
                                  'index': True,
                                  'nullable': False})
                ftype = attrs.get('type', 'string').lower()
                if ftype not in TYPE_MAP:
                    raise exceptions.SchemaError(
                        'field "%s" has unknown type "%s"' % (name, ftype))
                self.fields[name] = TYPE_MAP[ftype](self, name, attrs)
            else:
                raise exceptions.SchemaError(
                    'invalid field name "%s"' % name)
        if 'name' not in self.fields:
            raise exceptions.SchemaError('missing required "name" field')

    def to_net(self, item, ignore_missing=False):
        if isinstance(item, dict):
            attr_getter = lambda x, y: x.get(y)
        else:
{'fraction_non_alphanumeric': 0.11025517573423206, 'fraction_numerical': 0.001203659123736158, 'mean_word_length': 2.1750095529231945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6161363', 'n_tokens_mistral': 2382, 'n_tokens_neox': 2323, 'n_words': 725}