<commit_before>from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool, create_engine
from sqlalchemy.engine.url import URL
from logging.config import fileConfig

from os import getenv
import re

# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config

# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)

# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None

# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... config.

# database yaml file for connectivity info
db_yml = getenv('PERCEPTION_DB_INFO')

if db_yml is None:
    db_yml = 'perception/config/database.yml'

new_list = []
reg_clean = re.compile(r'[:]')

with open(db_yml, 'r') as f:
  new_list += [re.sub(r':\s', ':', line.strip()) for line in f if reg_clean.search(line)]
  config_info = dict(map(str, x.split(':')) for x in new_list)

db_info = {'drivername': config_info['drivername'],
           'username': config_info['username'],
           'password': config_info['password'],
           'host': config_info['host'],
           'database': config_info['database']}

# build URL
sqlalchemy_url = '%s://%s:%s@%s/%s' % (config_info['drivername'],
                                       config_info['username'],
                                       config_info['password'],
                                       config_info['host'],
                                       config_info['database'])


def run_migrations_offline():
    """Run migrations in 'offline' mode.

    This configures the context with just a URL
    and not an Engine, though an Engine is acceptable
    here as well.  By skipping the Engine creation
    we don't even need a DBAPI to be available.

    Calls to context.execute() here emit the given string to the
    script output.

    """
    url = config.get_main_option(sqlalchemy_url)
    context.configure(
        url=url, target_metadata=target_metadata, literal_binds=True)

    with context.begin_transaction():
        context.run_migrations()


def run_migrations_online():
    """Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context.

    """
    connectable = create_engine(URL(**db_info), pool_size=20)

    with connectable.connect() as connection:
        context.configure(
            connection=connection,
            target_metadata=target_metadata
        )

        with context.begin_transaction():
            context.run_migrations()

if context.is_offline_mode():
    run_migrations_offline()
else:
    run_migrations_online()<commit_msg>import database config from config/configuration<commit_after>from __future__ import with_statement
from alembic import context
from sqlalchemy import create_engine
from sqlalchemy.engine.url import URL
from logging.config import fileConfig
from ...config import configuration as myconfig

# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config

# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)

# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None

# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... config.

db_config = {'drivername': myconfig.db_drivername,
             'host': myconfig.db_host,
             'database': myconfig.database,
             'username': myconfig.db_username,
             'password': <PASSWORD>.db_password}

# build URL
sqlalchemy_url = '%s://%s:%s@%s/%s' % (myconfig.db_drivername,
                                       myconfig.db_username,
                                       myconfig.db_password,
                                       myconfig.db_host,
                                       myconfig.database)


def run_migrations_offline():
    """Run migrations in 'offline' mode.

    This configures the context with just a URL
    and not an Engine, though an Engine is acceptable
    here as well.  By skipping the Engine creation
    we don't even need a DBAPI to be available.

    Calls to context.execute() here emit the given string to the
    script output.

    """
    url = config.get_main_option(sqlalchemy_url)
    context.configure(
        url=url, target_metadata=target_metadata, literal_binds=True)

    with context.begin_transaction():
        context.run_migrations()


def run_migrations_online():
    """Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context.

    """
    connectable = create_engine(URL(**db_config), pool_size=20)

    with connectable.connect() as connection:
        context.configure(
            connection=connection,
            target_metadata=target_metadata
        )

        with context.begin_transaction():
            context.run_migrations()

if context.is_offline_mode():
    run_migrations_offline()
else:
    run_migrations_online()
{'fraction_non_alphanumeric': 0.08729441532622448, 'fraction_numerical': 0.0007229351165732876, 'mean_word_length': 2.9585121602288984, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7134174', 'n_tokens_mistral': 1562, 'n_tokens_neox': 1521, 'n_words': 477}