<commit_before>import abc
import uuid
import struct
import time

from bson import json_util, EPOCH_NAIVE
# Monkeypatch to not include tz_info in decoded JSON.
# Kind of a ridiculous solution, but works.
json_util.EPOCH_AWARE = EPOCH_NAIVE

from sqlalchemy import String, Text, event
from sqlalchemy.types import TypeDecorator, BINARY
from sqlalchemy.interfaces import PoolListener
from sqlalchemy.engine import Engine
from sqlalchemy.ext.mutable import Mutable
from sqlalchemy.ext.declarative import DeclarativeMeta

from inbox.util.encoding import base36encode, base36decode

from inbox.log import get_logger
log = get_logger()


SLOW_QUERY_THRESHOLD_MS = 5000
MAX_TEXT_LENGTH = 65535


@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement,
                          parameters, context, executemany):
    context._query_start_time = time.time()


@event.listens_for(Engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement,
                         parameters, context, executemany):
    total = int(1000 * (time.time() - context._query_start_time))
    # We only care about slow reads here
    if total > SLOW_QUERY_THRESHOLD_MS and statement.startswith('SELECT'):
        statement = ' '.join(statement.split())
        try:
            log.warning('slow query', query_time=total, statement=statement,
                        parameters=parameters)
        except UnicodeDecodeError:
            log.warning('slow query', query_time=total)
            log.error('logging UnicodeDecodeError')


class SQLAlchemyCompatibleAbstractMetaClass(DeclarativeMeta, abc.ABCMeta):
    """Declarative model classes that *also* inherit from an abstract base
    class need a metaclass like this one, in order to prevent metaclass
    conflict errors."""
    pass


class ABCMixin(object):
    """Use this if you want a mixin class which is actually an abstract base
    class, for example in order to enforce that concrete subclasses define
    particular methods or properties."""
    __metaclass__ = SQLAlchemyCompatibleAbstractMetaClass
    __abstract__ = True


# Column Types<commit_msg>Escape parameters in log statement to prevent downstream parse errors.<commit_after>import abc
import uuid
import struct
import time

from bson import json_util, EPOCH_NAIVE
# Monkeypatch to not include tz_info in decoded JSON.
# Kind of a ridiculous solution, but works.
json_util.EPOCH_AWARE = EPOCH_NAIVE

from sqlalchemy import String, Text, event
from sqlalchemy.types import TypeDecorator, BINARY
from sqlalchemy.interfaces import PoolListener
from sqlalchemy.engine import Engine
from sqlalchemy.ext.mutable import Mutable
from sqlalchemy.ext.declarative import DeclarativeMeta

from inbox.util.encoding import base36encode, base36decode

from inbox.log import get_logger
log = get_logger()


SLOW_QUERY_THRESHOLD_MS = 5000
MAX_TEXT_LENGTH = 65535


@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement,
                          parameters, context, executemany):
    context._query_start_time = time.time()


@event.listens_for(Engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement,
                         parameters, context, executemany):
    total = int(1000 * (time.time() - context._query_start_time))
    # We only care about slow reads here
    if total > SLOW_QUERY_THRESHOLD_MS and statement.startswith('SELECT'):
        statement = ' '.join(statement.split())
        try:
            # Turn `parameters` into a string as a quick fix for easier
            # downstream parsing (Elasticsearch has trouble with arrays of
            # heterogeneous type such as [1, 'a']).
            log.warning('slow query', query_time=total, statement=statement,
                        parameters=str(parameters))
        except UnicodeDecodeError:
            log.warning('slow query', query_time=total)
            log.error('logging UnicodeDecodeError')


class SQLAlchemyCompatibleAbstractMetaClass(DeclarativeMeta, abc.ABCMeta):
    """Declarative model classes that *also* inherit from an abstract base
    class need a metaclass like this one, in order to prevent metaclass
    conflict errors."""
    pass


class ABCMixin(object):
    """Use this if you want a mixin class which is actually an abstract base
    class, for example in order to enforce that concrete subclasses define
    particular methods or properties."""
    __metaclass__ = SQLAlchemyCompatibleAbstractMetaClass
    __abstract__ = True


# Column Types
{'fraction_non_alphanumeric': 0.06781885801793919, 'fraction_numerical': 0.007656967840735069, 'mean_word_length': 3.8227848101265822, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2985216', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1246, 'n_words': 409}