<commit_before>import re
from contextlib import contextmanager
from weakref import WeakSet

import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import HSTORE, INET
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.schema import CreateTable


__version__ = '0.1'


@compiles(CreateTable)
def _add_if_not_exists(element, compiler, **kw):
    output = compiler.visit_create_table(element, **kw)
    if element.element.info.get('ifexists'):
        output = re.sub(
            '^\s*(CREATE (TEMP )?TABLE)',
            '\\1 IF NOT EXISTS',
            output,
            re.DOTALL
        )
    return output


def raw_execute(conn, stmt):
    cursor = conn.connection.cursor()
    compiled = stmt.compile(conn.engine)
    return cursor.execute(str(compiled), compiled.params)


def read_file(file_):
    with open(file_) as f:
        s = f.read()
    return s.decode('utf8')


def activity_base(base):
    class ActivityBase(base):
        __abstract__ = True
        event_id = sa.Column(sa.BigInteger, primary_key=True)
        schema_name = sa.Column(sa.Text)
        table_name = sa.Column(sa.Text)
        relid = sa.Column(sa.Integer)
        session_user_name = sa.Column(sa.Text)
        issued_at = sa.Column(sa.DateTime)
        transaction_id = sa.Column(sa.BigInteger)
        application_name = sa.Column(sa.Text)
        client_addr = sa.Column(INET)
        client_port = sa.Column(sa.Integer)
        verb = sa.Column(sa.Text)
        actor_id = sa.Column(sa.Text)<commit_msg>Remove unnecessary decode<commit_after>import re
from contextlib import contextmanager
from weakref import WeakSet

import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import HSTORE, INET
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.schema import CreateTable


__version__ = '0.1'


@compiles(CreateTable)
def _add_if_not_exists(element, compiler, **kw):
    output = compiler.visit_create_table(element, **kw)
    if element.element.info.get('ifexists'):
        output = re.sub(
            '^\s*(CREATE (TEMP )?TABLE)',
            '\\1 IF NOT EXISTS',
            output,
            re.DOTALL
        )
    return output


def raw_execute(conn, stmt):
    cursor = conn.connection.cursor()
    compiled = stmt.compile(conn.engine)
    return cursor.execute(str(compiled), compiled.params)


def read_file(file_):
    with open(file_) as f:
        s = f.read()
    return s


def activity_base(base):
    class ActivityBase(base):
        __abstract__ = True
        event_id = sa.Column(sa.BigInteger, primary_key=True)
        schema_name = sa.Column(sa.Text)
        table_name = sa.Column(sa.Text)
        relid = sa.Column(sa.Integer)
        session_user_name = sa.Column(sa.Text)
        issued_at = sa.Column(sa.DateTime)
        transaction_id = sa.Column(sa.BigInteger)
        application_name = sa.Column(sa.Text)
        client_addr = sa.Column(INET)
        client_port = sa.Column(sa.Integer)
        verb = sa.Column(sa.Text)
        actor_id = sa.Column(sa.Text)
{'fraction_non_alphanumeric': 0.10698906922822127, 'fraction_numerical': 0.0023186485591255384, 'mean_word_length': 2.8520408163265305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3666945', 'n_tokens_mistral': 977, 'n_tokens_neox': 953, 'n_words': 207}