<commit_before>import ConfigParser
import os
import unittest
from pyramid.config import Configurator
from pyramid import testing

global_config = {}

ini = ConfigParser.ConfigParser()

# we need to pull the right ini for the test we want to run
# by default pullup test.ini, but we might want to test mysql, pgsql, etc
test_ini = os.environ.get('BOOKIE_TEST_INI', None)
if test_ini:
    ini.read(test_ini)
else:
    ini.read('test.ini')

settings = dict(ini.items('app:bookie'))


def setup_db(settings):
    """ We need to create the test sqlite database to run our tests against

    If the db exists, remove it
    We're using the SA-Migrations API to create the db and catch it up to the
    latest migration level for testing

    In theory, we could use this API to do version specific testing as well if
    we needed to.

    If we want to run any tests with a fresh db we can call this function

    """
    from migrate.versioning import api as mig
    sa_url = settings['sqlalchemy.url']
    migrate_repository = 'migrations'

    # we're hackish here since we're going to assume the test db is whatever is
    # after the last slash of the SA url sqlite:///somedb.db
    db_name = sa_url[sa_url.rindex('/') + 1:]
    try:
        # if this is a sqlite db then try to take care of the db file
        os.remove(db_name)
        open(db_name, 'w').close()
    except:
        pass

    mig.version_control(sa_url, migrate_repository)
    mig.upgrade(sa_url, migrate_repository)
<commit_msg>Make sure we drop/empty old tables when running tests<commit_after>import ConfigParser
import os
import unittest
from pyramid.config import Configurator
from pyramid import testing

global_config = {}

ini = ConfigParser.ConfigParser()

# we need to pull the right ini for the test we want to run
# by default pullup test.ini, but we might want to test mysql, pgsql, etc
test_ini = os.environ.get('BOOKIE_TEST_INI', None)
if test_ini:
    ini.read(test_ini)
else:
    ini.read('test.ini')

settings = dict(ini.items('app:bookie'))


def setup_db(settings):
    """ We need to create the test sqlite database to run our tests against

    If the db exists, remove it
    We're using the SA-Migrations API to create the db and catch it up to the
    latest migration level for testing

    In theory, we could use this API to do version specific testing as well if
    we needed to.

    If we want to run any tests with a fresh db we can call this function

    """
    from migrate.versioning import api as mig
    sa_url = settings['sqlalchemy.url']
    migrate_repository = 'migrations'

    if 'mysql' in sa_url:
        # MYSQL CONFIG
        from sqlalchemy import create_engine
        engine = create_engine(sa_url)

        # # drop any existing data
        all_tables = engine.execute('SHOW TABLES');
        if all_tables.rowcount:
            qry = "`, `".join([res[0] for res in all_tables])
            engine.execute("DROP TABLES `" + qry + '`;')

    elif 'postgres' in sa_url:
        # MYSQL CONFIG
        from sqlalchemy import create_engine
        engine = create_engine(sa_url)

        # # drop any existing data
        all_tables = engine.execute("""SELECT table_name
                                       FROM information_schema.tables
                                       WHERE table_schema = 'public'
        """);

        if all_tables.rowcount:
            qry = ", ".join([res[0] for res in all_tables])
            engine.execute("DROP TABLE " + qry + ';')


    else:
        # we're hackish here since we're going to assume the test db is whatever is
        # after the last slash of the SA url sqlite:///somedb.db
        db_name = sa_url[sa_url.rindex('/') + 1:]
        try:
            # if this is a sqlite db then try to take care of the db file
            os.remove(db_name)
            open(db_name, 'w').close()
        except:
            pass

    mig.version_control(sa_url, migrate_repository)
    mig.upgrade(sa_url, migrate_repository)

{'fraction_non_alphanumeric': 0.07865731462925851, 'fraction_numerical': 0.001002004008016032, 'mean_word_length': 2.6333030027297544, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '643691', 'n_tokens_mistral': 1213, 'n_tokens_neox': 1158, 'n_words': 454}