<commit_before>import logging

from zope.dottedname.resolve import resolve

from .documents import (
    BaseDocument, ESBaseDocument, BaseMixin,
    get_document_cls, get_document_classes)
from .serializers import JSONEncoder, ESJSONSerializer
from .signals import ESMetaclass
from .utils import (
    relationship_fields, is_relationship_field,
    get_relationship_cls, FieldData)
from .fields import (
    BigIntegerField,
    BooleanField,
    DateField,
    DateTimeField,
    ChoiceField,
    FloatField,
    IntegerField,
    IntervalField,
    BinaryField,
    DecimalField,
    PickleField,
    SmallIntegerField,
    StringField,
    TextField,
    TimeField,
    UnicodeField,
    UnicodeTextField,
    Relationship,
    IdField,
    ForeignKeyField,
    DictField,
    ListField,
)

from .fields import ProcessableRelationshipProperty as RelationshipField


log = logging.getLogger(__name__)


def includeme(config):
    """ Include required packages. """
    config.include('pyramid_tm')
    config.include('pyramid_sqlalchemy')


def setup_database(config):
    """ Setup db engine, db itself. Create db if it doesn't exist. """
    from sqlalchemy import engine_from_config
    from sqlalchemy_utils import database_exists, create_database
    from pyramid_sqlalchemy import BaseObject
    engine = engine_from_config(config.registry.settings, 'sqlalchemy.')
    BaseObject.metadata.bind = engine
    if not database_exists(engine.url):
        log.info(
            'Database does not exit. Creating database at %s' % engine.url)
        create_database(engine.url)

    # Create HSTORE extension if database is postgresql
    if engine.url.get_backend_name() == 'postgresql':
        engine.execute('CREATE EXTENSION IF NOT EXISTS hstore;')

    BaseObject.metadata.create_all(engine)<commit_msg>typo: s/exit/exist<commit_after>import logging

from zope.dottedname.resolve import resolve

from .documents import (
    BaseDocument, ESBaseDocument, BaseMixin,
    get_document_cls, get_document_classes)
from .serializers import JSONEncoder, ESJSONSerializer
from .signals import ESMetaclass
from .utils import (
    relationship_fields, is_relationship_field,
    get_relationship_cls, FieldData)
from .fields import (
    BigIntegerField,
    BooleanField,
    DateField,
    DateTimeField,
    ChoiceField,
    FloatField,
    IntegerField,
    IntervalField,
    BinaryField,
    DecimalField,
    PickleField,
    SmallIntegerField,
    StringField,
    TextField,
    TimeField,
    UnicodeField,
    UnicodeTextField,
    Relationship,
    IdField,
    ForeignKeyField,
    DictField,
    ListField,
)

from .fields import ProcessableRelationshipProperty as RelationshipField


log = logging.getLogger(__name__)


def includeme(config):
    """ Include required packages. """
    config.include('pyramid_tm')
    config.include('pyramid_sqlalchemy')


def setup_database(config):
    """ Setup db engine, db itself. Create db if it doesn't exist. """
    from sqlalchemy import engine_from_config
    from sqlalchemy_utils import database_exists, create_database
    from pyramid_sqlalchemy import BaseObject
    engine = engine_from_config(config.registry.settings, 'sqlalchemy.')
    BaseObject.metadata.bind = engine
    if not database_exists(engine.url):
        log.info(
            'Database does not exist. Creating database at %s' % engine.url)
        create_database(engine.url)

    # Create HSTORE extension if database is postgresql
    if engine.url.get_backend_name() == 'postgresql':
        engine.execute('CREATE EXTENSION IF NOT EXISTS hstore;')

    BaseObject.metadata.create_all(engine)
{'fraction_non_alphanumeric': 0.07402311502476609, 'fraction_numerical': 0.0, 'mean_word_length': 3.795514511873351, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4196925', 'n_tokens_mistral': 1076, 'n_tokens_neox': 1024, 'n_words': 264}