<commit_before>from __future__ import with_statement

from sqlalchemy.engine import url, default
import shutil
import os
import itertools
from sqlalchemy import create_engine, text, MetaData
from alembic import util
from alembic.migration import MigrationContext
from alembic.environment import EnvironmentContext
import re
import alembic
from alembic.operations import Operations
from alembic.script import ScriptDirectory, Script
from alembic import ddl
import StringIO
from alembic.ddl.impl import _impls
import ConfigParser
from nose import SkipTest
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.util import decorator
import shutil
import textwrap

staging_directory = os.path.join(os.path.dirname(__file__), 'scratch')
files_directory = os.path.join(os.path.dirname(__file__), 'files')

testing_config = ConfigParser.ConfigParser()
testing_config.read(['test.cfg'])

def sqlite_db():
    # sqlite caches table pragma info
    # per connection, so create a new
    # engine for each assertion
    dir_ = os.path.join(staging_directory, 'scripts')
    return create_engine('sqlite:///%s/foo.db' % dir_)

def capture_db():
    buf = []
    def dump(sql, *multiparams, **params):
        buf.append(str(sql.compile(dialect=engine.dialect)))
    engine = create_engine("postgresql://", strategy="mock", executor=dump)
    return engine, buf

_engs = {}
def db_for_dialect(name):
    if name in _engs:
        return _engs[name]
    else:
        try:
            cfg = testing_config.get("db", name)
        except ConfigParser.NoOptionError:
            raise SkipTest("No dialect %r in test.cfg" % name)
        try:
            eng = create_engine(cfg, echo=True)
        except ImportError, er1:
            raise SkipTest("Can't import DBAPI: %s" % er1)
        try:
            conn = eng.connect()
        except SQLAlchemyError, er2:
            raise SkipTest("Can't connect to database: %s" % er2)
        _engs[name] = eng
        return eng

@decorator
def requires_07(fn, *arg, **kw):
    if not util.sqla_07:
        raise SkipTest("SQLAlchemy 0.7 required")
    return fn(*arg, **kw)

@decorator
def requires_08(fn, *arg, **kw):
    if not util.sqla_08:
        raise SkipTest("SQLAlchemy 0.8.0b2 or greater required")
    return fn(*arg, **kw)

_dialects = {}
def _get_dialect(name):
    if name is None or name == 'default':
        return default.DefaultDialect()
    else:
        try:
            return _dialects[name]
        except KeyError:
            dialect_mod = getattr(__import__('sqlalchemy.dialects.%s' % name).dialects, name)<commit_msg>remove this echo<commit_after>from __future__ import with_statement

from sqlalchemy.engine import url, default
import shutil
import os
import itertools
from sqlalchemy import create_engine, text, MetaData
from alembic import util
from alembic.migration import MigrationContext
from alembic.environment import EnvironmentContext
import re
import alembic
from alembic.operations import Operations
from alembic.script import ScriptDirectory, Script
from alembic import ddl
import StringIO
from alembic.ddl.impl import _impls
import ConfigParser
from nose import SkipTest
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.util import decorator
import shutil
import textwrap

staging_directory = os.path.join(os.path.dirname(__file__), 'scratch')
files_directory = os.path.join(os.path.dirname(__file__), 'files')

testing_config = ConfigParser.ConfigParser()
testing_config.read(['test.cfg'])

def sqlite_db():
    # sqlite caches table pragma info
    # per connection, so create a new
    # engine for each assertion
    dir_ = os.path.join(staging_directory, 'scripts')
    return create_engine('sqlite:///%s/foo.db' % dir_)

def capture_db():
    buf = []
    def dump(sql, *multiparams, **params):
        buf.append(str(sql.compile(dialect=engine.dialect)))
    engine = create_engine("postgresql://", strategy="mock", executor=dump)
    return engine, buf

_engs = {}
def db_for_dialect(name):
    if name in _engs:
        return _engs[name]
    else:
        try:
            cfg = testing_config.get("db", name)
        except ConfigParser.NoOptionError:
            raise SkipTest("No dialect %r in test.cfg" % name)
        try:
            eng = create_engine(cfg)
        except ImportError, er1:
            raise SkipTest("Can't import DBAPI: %s" % er1)
        try:
            conn = eng.connect()
        except SQLAlchemyError, er2:
            raise SkipTest("Can't connect to database: %s" % er2)
        _engs[name] = eng
        return eng

@decorator
def requires_07(fn, *arg, **kw):
    if not util.sqla_07:
        raise SkipTest("SQLAlchemy 0.7 required")
    return fn(*arg, **kw)

@decorator
def requires_08(fn, *arg, **kw):
    if not util.sqla_08:
        raise SkipTest("SQLAlchemy 0.8.0b2 or greater required")
    return fn(*arg, **kw)

_dialects = {}
def _get_dialect(name):
    if name is None or name == 'default':
        return default.DefaultDialect()
    else:
        try:
            return _dialects[name]
        except KeyError:
            dialect_mod = getattr(__import__('sqlalchemy.dialects.%s' % name).dialects, name)
{'fraction_non_alphanumeric': 0.09887553315238465, 'fraction_numerical': 0.006979449398991857, 'mean_word_length': 3.3646362098138747, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6228502', 'n_tokens_mistral': 1645, 'n_tokens_neox': 1589, 'n_words': 456}