<commit_before>import abc
import six
import uuid
import re
import traceback
import logging

logger = logging.getLogger(__name__)

from types import LambdaType
from collections import defaultdict

from ...document import Document
from ..base import Backend as BaseBackend
from ..base import NotInTransaction,DoNotSerialize
from ..file.serializers import JsonSerializer
from .queryset import QuerySet
from .relations import ManyToManyProxy

from blitzdb.fields import (ForeignKeyField,
                            ManyToManyField,
                            OneToManyField,
                            CharField,
                            EnumField,
                            IntegerField,
                            TextField,
                            FloatField,
                            BooleanField,
                            BinaryField,
                            DateField,
                            DateTimeField,
                            BaseField
                            )

from sqlalchemy.exc import IntegrityError
from sqlalchemy.schema import MetaData,Table,Column,ForeignKey,UniqueConstraint
from sqlalchemy.types import (Integer,
                              VARCHAR,
                              String,
                              Float,
                              Enum,
                              Boolean,
                              Date,
                              DateTime,
                              Text,
                              LargeBinary,
                              Unicode)
from sqlalchemy.sql import select,insert,update,func,and_,or_,not_,expression,null
from sqlalchemy.ext.compiler import compiles
from blitzdb.helpers import get_value, set_value, delete_value

@compiles(DateTime, "sqlite")
def compile_binary_sqlite(type_, compiler, **kw):
    return "VARCHAR(64)"

class ExcludedFieldsEncoder(object):

    def __init__(self,backend,collection):
        self.collection = collection
        self.backend = backend

    def encode(self,obj,path = []):
        if not path:
            return obj
        key = ".".join([str(p) for p in path])
        if key in self.backend._excluded_keys[self.collection]:
            raise DoNotSerialize
        return obj

class Backend(BaseBackend):

    """
    A SQL backend.

    :param db: An instance of a `sqlalchemy. 
    <http://www.sqlalchemy.org>`_ class

    Example usage:

    .. code-block:: python

        from sqlalchemy import create_engine
        from blitzdb.backends.sql import Backend as SQLBackend

        my_engine = create_engine(...)

        #create a new BlitzDB backend using a SQLAlchemy engine
        backend = SQLBackend(my_engine)
    """

    class Meta(BaseBackend.Meta):
        pass

    def __init__(self, engine, table_postfix = '',create_schema = False,**kwargs):
        super(Backend, self).__init__(**kwargs)

        self._engine_getter = engine
        self._engine = None
        self._schema_initialized = False
        self._relationship_classes = []
        self._transactions = []
        self.table_postfix = table_postfix

        if create_schema:
            self.create_schema()

        self._conn = None
        self._auto_transaction = False

    @property
    def engine(self):
        if self._engine is None:<commit_msg>Auto-initialize schema.<commit_after>import abc
import six
import uuid
import re
import traceback
import logging

logger = logging.getLogger(__name__)

from types import LambdaType
from collections import defaultdict

from ...document import Document
from ..base import Backend as BaseBackend
from ..base import NotInTransaction,DoNotSerialize
from ..file.serializers import JsonSerializer
from .queryset import QuerySet
from .relations import ManyToManyProxy

from blitzdb.fields import (ForeignKeyField,
                            ManyToManyField,
                            OneToManyField,
                            CharField,
                            EnumField,
                            IntegerField,
                            TextField,
                            FloatField,
                            BooleanField,
                            BinaryField,
                            DateField,
                            DateTimeField,
                            BaseField
                            )

from sqlalchemy.exc import IntegrityError
from sqlalchemy.schema import MetaData,Table,Column,ForeignKey,UniqueConstraint
from sqlalchemy.types import (Integer,
                              VARCHAR,
                              String,
                              Float,
                              Enum,
                              Boolean,
                              Date,
                              DateTime,
                              Text,
                              LargeBinary,
                              Unicode)
from sqlalchemy.sql import select,insert,update,func,and_,or_,not_,expression,null
from sqlalchemy.ext.compiler import compiles
from blitzdb.helpers import get_value, set_value, delete_value

@compiles(DateTime, "sqlite")
def compile_binary_sqlite(type_, compiler, **kw):
    return "VARCHAR(64)"

class ExcludedFieldsEncoder(object):

    def __init__(self,backend,collection):
        self.collection = collection
        self.backend = backend

    def encode(self,obj,path = []):
        if not path:
            return obj
        key = ".".join([str(p) for p in path])
        if key in self.backend._excluded_keys[self.collection]:
            raise DoNotSerialize
        return obj

class Backend(BaseBackend):

    """
    A SQL backend.

    :param db: An instance of a `sqlalchemy. 
    <http://www.sqlalchemy.org>`_ class

    Example usage:

    .. code-block:: python

        from sqlalchemy import create_engine
        from blitzdb.backends.sql import Backend as SQLBackend

        my_engine = create_engine(...)

        #create a new BlitzDB backend using a SQLAlchemy engine
        backend = SQLBackend(my_engine)
    """

    class Meta(BaseBackend.Meta):
        pass

    def __init__(self, engine, table_postfix = '',create_schema = False,**kwargs):
        super(Backend, self).__init__(**kwargs)

        self._engine_getter = engine
        self._engine = None
        self._schema_initialized = False
        self._relationship_classes = []
        self._transactions = []
        self.table_postfix = table_postfix

        self.init_schema()

        if create_schema:
            self.create_schema()

        self._conn = None
        self._auto_transaction = False

    @property
    def engine(self):
        if self._engine is None:
{'fraction_non_alphanumeric': 0.06635570267830274, 'fraction_numerical': 0.0006018657839301836, 'mean_word_length': 1.6345620293301626, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6516166', 'n_tokens_mistral': 1637, 'n_tokens_neox': 1593, 'n_words': 427}