<commit_before>"""
Run migrations

By default runs all migrations in order. If arguments are provided then
only migration files that match the arguments are run.

For example:

python run_migrations.py 001_add_week_start.py

Would only run the first migration.
"""
import imp
import os
import sys
from os.path import join
import logging
from backdrop.core.database import Database

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)

ROOT_PATH = os.path.abspath(os.path.dirname(__file__))


def load_config(env):
    config_path = os.path.join(ROOT_PATH, 'backdrop', 'write', 'config')
    fp = None
    try:
        sys.path.append(config_path)
        fp, pathname, description = imp.find_module(
            "backdrop/write/config/%s" % env)
        return imp.load_module(env, fp, pathname, description)
    finally:
        sys.path.pop()
        if fp:
            fp.close()


def get_database(config):
    return Database(
        config.MONGO_HOSTS, config.MONGO_PORT, config.DATABASE_NAME)


def get_migrations(migration_files):
    migrations_path = join(ROOT_PATH, 'migrations')
    for migration_file in os.listdir(migrations_path):
        if migration_files is None or migration_file in migration_files:
            migration_path = join(migrations_path, migration_file)

            yield imp.load_source('migration', migration_path)
<commit_msg>Only run .py migration files<commit_after>"""
Run migrations

By default runs all migrations in order. If arguments are provided then
only migration files that match the arguments are run.

For example:

python run_migrations.py 001_add_week_start.py

Would only run the first migration.
"""
import imp
import os
import sys
from os.path import join
import logging
from backdrop.core.database import Database

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)

ROOT_PATH = os.path.abspath(os.path.dirname(__file__))


def load_config(env):
    config_path = os.path.join(ROOT_PATH, 'backdrop', 'write', 'config')
    fp = None
    try:
        sys.path.append(config_path)
        fp, pathname, description = imp.find_module(
            "backdrop/write/config/%s" % env)
        return imp.load_module(env, fp, pathname, description)
    finally:
        sys.path.pop()
        if fp:
            fp.close()


def get_database(config):
    return Database(
        config.MONGO_HOSTS, config.MONGO_PORT, config.DATABASE_NAME)


def get_migrations(migration_files):
    migrations_path = join(ROOT_PATH, 'migrations')
    for migration_file in os.listdir(migrations_path):
        if not migration_file.endswith('.py'):
            continue
        if migration_files is None or migration_file in migration_files:
            migration_path = join(migrations_path, migration_file)

            yield imp.load_source('migration', migration_path)

{'fraction_non_alphanumeric': 0.08756567425569177, 'fraction_numerical': 0.0021015761821366026, 'mean_word_length': 3.533333333333333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4180584', 'n_tokens_mistral': 861, 'n_tokens_neox': 863, 'n_words': 234}