<commit_before>import json

from alchimia import TWISTED_STRATEGY
from sqlalchemy import MetaData, Table, Column, String, Text, create_engine
from sqlalchemy.schema import CreateTable
from twisted.internet.defer import succeed


def get_engine(conn_str, reactor):
    return create_engine(conn_str, reactor=reactor, strategy=TWISTED_STRATEGY)


class TableMissingError(Exception):
    pass


class CollectionMissingError(Exception):
    pass


class make_table(object):
    def __init__(self, *args, **kw):
        self.args = args
        self.kw = kw

    def make_table(self, name, metadata):
        return Table(name, metadata, *self.copy_args(), **self.kw)

    def copy_args(self):
        for arg in self.args:
            if isinstance(arg, Column):
                yield arg.copy()
            else:
                yield arg


def _false_to_error(result, err):
    if not result:
        raise err
    return result


class _PrefixedTables(object):
    def __init__(self, name, connection):
        self.name = name
        self._conn = connection
        self._metadata = MetaData()
        for attr in dir(self):
            attrval = getattr(self, attr)
            if isinstance(attrval, make_table):
                setattr(self, attr, attrval.make_table(
                    self.get_table_name(attr), self._metadata))

    def get_table_name(self, name):
        raise NotImplementedError(
            "_PrefixedTables should not be used directly.")

    def _create_table(self, trx, table):
        # This works around alchimia's current inability to create tables only
        # if they don't already exist.

        table_exists_err_templates = [
            'table %(name)s already exists',
            'table "%(name)s" already exists',
            'relation %(name)s already exists',
            'relation "%(name)s" already exists',
        ]

        def table_exists_errback(f):
            for err_template in table_exists_err_templates:
                if err_template % {'name': table.name} in str(f.value):
                    return None
            return f

        d = self._conn.execute(CreateTable(table))
        d.addErrback(table_exists_errback)
        return d.addCallback(lambda r: trx)

    def _create_tables(self):
        d = self._conn.begin()
        for table in self._metadata.sorted_tables:
            d.addCallback(self._create_table, table)
        return d.addCallback(lambda trx: trx.commit())

    def exists(self):
        raise NotImplementedError(
            "_PrefixedTables should not be used directly.")

    def _execute_query(self, query, *args, **kw):
        return self._conn.execute(query, *args, **kw)

    def execute_query(self, query, *args, **kw):
        raise NotImplementedError(
            "_PrefixedTables should not be used directly.")

    def execute_fetchall(self, query, *args, **kw):
        d = self.execute_query(query, *args, **kw)
        return d.addCallback(lambda result: result.fetchall())


class CollectionMetadata(_PrefixedTables):
    """
    Metadata manager for PrefixedTableCollection.

    This tracks table prefixes and metadata for a given collection type.
    """

    collection_metadata = make_table(
        Column("name", String(512), primary_key=True),
        Column("metadata_json", Text(), nullable=False),
    )

    _metadata_cache_dict = None
<commit_msg>MySQL is making me sad.<commit_after>import json

from alchimia import TWISTED_STRATEGY
from sqlalchemy import MetaData, Table, Column, String, Text, create_engine
from sqlalchemy.schema import CreateTable
from twisted.internet.defer import succeed


def get_engine(conn_str, reactor):
    return create_engine(conn_str, reactor=reactor, strategy=TWISTED_STRATEGY)


class TableMissingError(Exception):
    pass


class CollectionMissingError(Exception):
    pass


class make_table(object):
    def __init__(self, *args, **kw):
        self.args = args
        self.kw = kw

    def make_table(self, name, metadata):
        return Table(name, metadata, *self.copy_args(), **self.kw)

    def copy_args(self):
        for arg in self.args:
            if isinstance(arg, Column):
                yield arg.copy()
            else:
                yield arg


def _false_to_error(result, err):
    if not result:
        raise err
    return result


class _PrefixedTables(object):
    def __init__(self, name, connection):
        self.name = name
        self._conn = connection
        self._metadata = MetaData()
        for attr in dir(self):
            attrval = getattr(self, attr)
            if isinstance(attrval, make_table):
                setattr(self, attr, attrval.make_table(
                    self.get_table_name(attr), self._metadata))

    def get_table_name(self, name):
        raise NotImplementedError(
            "_PrefixedTables should not be used directly.")

    def _create_table(self, trx, table):
        # This works around alchimia's current inability to create tables only
        # if they don't already exist.

        table_exists_err_templates = [
            'table %(name)s already exists',
            'table "%(name)s" already exists',
            'relation %(name)s already exists',
            'relation "%(name)s" already exists',
        ]

        def table_exists_errback(f):
            for err_template in table_exists_err_templates:
                if err_template % {'name': table.name} in str(f.value):
                    return None
            return f

        d = self._conn.execute(CreateTable(table))
        d.addErrback(table_exists_errback)
        return d.addCallback(lambda r: trx)

    def _create_tables(self):
        d = self._conn.begin()
        for table in self._metadata.sorted_tables:
            d.addCallback(self._create_table, table)
        return d.addCallback(lambda trx: trx.commit())

    def exists(self):
        raise NotImplementedError(
            "_PrefixedTables should not be used directly.")

    def _execute_query(self, query, *args, **kw):
        return self._conn.execute(query, *args, **kw)

    def execute_query(self, query, *args, **kw):
        raise NotImplementedError(
            "_PrefixedTables should not be used directly.")

    def execute_fetchall(self, query, *args, **kw):
        d = self.execute_query(query, *args, **kw)
        return d.addCallback(lambda result: result.fetchall())


class CollectionMetadata(_PrefixedTables):
    """
    Metadata manager for PrefixedTableCollection.

    This tracks table prefixes and metadata for a given collection type.
    """

    collection_metadata = make_table(
        Column("name", String(256), primary_key=True),
        Column("metadata_json", Text(), nullable=False),
    )

    _metadata_cache_dict = None

{'fraction_non_alphanumeric': 0.0942865600947306, 'fraction_numerical': 0.0008880994671403197, 'mean_word_length': 2.5827147401908803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2083755', 'n_tokens_mistral': 1954, 'n_tokens_neox': 1867, 'n_words': 528}