<commit_before>from __future__ import absolute_import, unicode_literals
import re
import copy
import collections
from sqlbuilder import smartsql
from . import signals
from .connections import get_db
from .utils import classproperty

try:
    str = unicode  # Python 2.* compatible
    string_types = (basestring,)
    integer_types = (int, long)
except NameError:
    string_types = (str,)
    integer_types = (int,)

SMARTSQL_DIALECTS = {
    'sqlite3': 'sqlite',
    'mysql': 'mysql',
    'postgresql': 'postgres',
    'postgresql_psycopg2': 'postgres',
    'postgis': 'postgres',
    'oracle': 'oracle',
}

cr = copy.copy(smartsql.cr)


class ModelNotRegistered(Exception):
    pass


class ModelRegistry(dict):

    def add(self, model):
        self[model._meta.name] = model

    def __getitem__(self, model_name):
        try:
            return self[model_name]
        except KeyError:
            raise ModelNotRegistered

registry = ModelRegistry()


class Field(object):
    def __init__(self, **kw):
        for k, v in kw.items():
            setattr(self, k, v)

    def to_python(self, value):
        return data_registry.convert_to_python(self.model.qs.dialect(), self.type_code, value)

    def to_string(self, value):
        data_registry.convert_to_string(value)


class ModelOptions(object):
    """Model options"""

    pk = 'id'
    using = 'default'
    field_class = Field

    def __init__(self, model, **kw):
        """Instance constructor"""
        self.relations = {}

        for k, v in kw:
            setattr(self, k, v)

        self.model = model
        if not hasattr(self, 'name'):
            self.name = ".".join((self.model.__module__, self.model.__name__))
        if not hasattr(self, 'db_table'):
            self.db_table = "_".join([
                re.sub(r"[^a-z0-9]", "", i.lower())
                for i in (self.model.__module__.split(".") + [self.model.__name__, ])
            ])

        if not hasattr(self, 'validations'):
            self.validations = {}
        for k, v in self.validations.items():
            if not isinstance(v, (list, tuple)):
                self.validations[k] = (v, )

        db = get_db(self.using)

        schema = db.describe_table(self.db_table)
        map = dict([(v, k) for k, v in getattr(self, 'map', {}).items()])
        # fileds and columns can be a descriptor for multilingual mapping.

        self.declared_fields = {}
        for name in self.model.__dict__:
            field = getattr(self.model, name, None)
            if isinstance(field, Field):
                self.declared_fields[name] = field
                delattr(self.model, name)
                if getattr(field, 'column', None):
                    map[field.column] = name

        # self.all(whole, total)_fields = collections.OrderedDict()  # with parents, MTI
        self.fields = collections.OrderedDict()
        self.columns = collections.OrderedDict()
        q = db.execute('SELECT * FROM {0} LIMIT 1'.format(qn(self.db_table)))
        # See cursor.description http://www.python.org/dev/peps/pep-0249/
        for row in q.description:
            column = row[0]
            name = map.get(column, column)
            data = schema.get(column, {})
            data.update({'column': column, 'type_code': row[1]})
            if name in self.declared_fields:
                field = self.declared_fields.get(name)
                field.__dict__.update(data)
            else:
                field = self.field_class(**data)
            self.add_field(field, name)

    def add_field(self, field, name):
        field.name = name
        field.model = self.model
        if getattr(field, 'validators', None):
            self.validations[name] = field.validators
        self.fields[name] = field
        self.columns[field.column] = field


class ModelBase(type):
    """Metaclass for Model"""
    options_class = ModelOptions

    def __new__(cls, name, bases, attrs):
        if name in ('Model', 'NewBase', ):
            return super(ModelBase, cls).__new__(cls, name, bases, attrs)

        new_cls = type.__new__(cls, name, bases, attrs)

        if hasattr(new_cls, 'Meta'):
            if isinstance(new_cls.Meta, new_cls.options_class):
                NewOptions = new_cls.Meta
            else:
                class NewOptions(new_cls.Meta, new_cls.options_class):
                    pass
        else:
            NewOptions = new_cls.options_class
        opts = new_cls._meta = NewOptions(new_cls)

        for key, rel in new_cls.__dict__.items():
            if isinstance(rel, Relation):
                rel.add_to_class(new_cls, key)

        registry.add(new_cls)

        for m in registry.values():
            for key, rel in m._meta.relations.items():
                try:
                    if hasattr(rel, 'add_related') and rel.rel_model is new_cls:<commit_msg>Meta.abstract<commit_after>from __future__ import absolute_import, unicode_literals
import re
import copy
import collections
from sqlbuilder import smartsql
from . import signals
from .connections import get_db
from .utils import classproperty

try:
    str = unicode  # Python 2.* compatible
    string_types = (basestring,)
    integer_types = (int, long)
except NameError:
    string_types = (str,)
    integer_types = (int,)

SMARTSQL_DIALECTS = {
    'sqlite3': 'sqlite',
    'mysql': 'mysql',
    'postgresql': 'postgres',
    'postgresql_psycopg2': 'postgres',
    'postgis': 'postgres',
    'oracle': 'oracle',
}

cr = copy.copy(smartsql.cr)


class ModelNotRegistered(Exception):
    pass


class ModelRegistry(dict):

    def add(self, model):
        self[model._meta.name] = model

    def __getitem__(self, model_name):
        try:
            return self[model_name]
        except KeyError:
            raise ModelNotRegistered

registry = ModelRegistry()


class Field(object):
    def __init__(self, **kw):
        for k, v in kw.items():
            setattr(self, k, v)

    def to_python(self, value):
        return data_registry.convert_to_python(self.model.qs.dialect(), self.type_code, value)

    def to_string(self, value):
        data_registry.convert_to_string(value)


class ModelOptions(object):
    """Model options"""

    pk = 'id'
    using = 'default'
    field_class = Field

    def __init__(self, model, **kw):
        """Instance constructor"""
        self.relations = {}

        for k, v in kw:
            setattr(self, k, v)

        self.model = model
        if not hasattr(self, 'name'):
            self.name = ".".join((self.model.__module__, self.model.__name__))
        if not hasattr(self, 'db_table'):
            self.db_table = "_".join([
                re.sub(r"[^a-z0-9]", "", i.lower())
                for i in (self.model.__module__.split(".") + [self.model.__name__, ])
            ])

        if not hasattr(self, 'validations'):
            self.validations = {}
        for k, v in self.validations.items():
            if not isinstance(v, (list, tuple)):
                self.validations[k] = (v, )

        db = get_db(self.using)

        schema = db.describe_table(self.db_table)
        map = dict([(v, k) for k, v in getattr(self, 'map', {}).items()])
        # fileds and columns can be a descriptor for multilingual mapping.

        self.declared_fields = {}
        for name in self.model.__dict__:
            field = getattr(self.model, name, None)
            if isinstance(field, Field):
                self.declared_fields[name] = field
                delattr(self.model, name)
                if getattr(field, 'column', None):
                    map[field.column] = name

        # self.all(whole, total)_fields = collections.OrderedDict()  # with parents, MTI
        self.fields = collections.OrderedDict()
        self.columns = collections.OrderedDict()
        q = db.execute('SELECT * FROM {0} LIMIT 1'.format(qn(self.db_table)))
        # See cursor.description http://www.python.org/dev/peps/pep-0249/
        for row in q.description:
            column = row[0]
            name = map.get(column, column)
            data = schema.get(column, {})
            data.update({'column': column, 'type_code': row[1]})
            if name in self.declared_fields:
                field = self.declared_fields.get(name)
                field.__dict__.update(data)
            else:
                field = self.field_class(**data)
            self.add_field(field, name)

    def add_field(self, field, name):
        field.name = name
        field.model = self.model
        if getattr(field, 'validators', None):
            self.validations[name] = field.validators
        self.fields[name] = field
        self.columns[field.column] = field


class ModelBase(type):
    """Metaclass for Model"""
    options_class = ModelOptions

    def __new__(cls, name, bases, attrs):

        new_cls = type.__new__(cls, name, bases, attrs)

        if name in ('Model', 'NewBase', ):
            return new_cls

        if getattr(attrs.get('Meta'), 'abstract', None):
            del new_cls.Meta
            return new_cls

        if hasattr(new_cls, 'Meta'):
            if isinstance(new_cls.Meta, new_cls.options_class):
                NewOptions = new_cls.Meta
            else:
                class NewOptions(new_cls.Meta, new_cls.options_class):
                    pass
        else:
            NewOptions = new_cls.options_class
        opts = new_cls._meta = NewOptions(new_cls)

        for key, rel in new_cls.__dict__.items():
            if isinstance(rel, Relation):
                rel.add_to_class(new_cls, key)

        registry.add(new_cls)

        for m in registry.values():
            for key, rel in m._meta.relations.items():
                try:
                    if hasattr(rel, 'add_related') and rel.rel_model is new_cls:
{'fraction_non_alphanumeric': 0.11945010183299389, 'fraction_numerical': 0.0026476578411405295, 'mean_word_length': 2.1387024608501117, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '180879', 'n_tokens_mistral': 3015, 'n_tokens_neox': 2896, 'n_words': 709}