<commit_before>import functools
import itertools
import json
import logging
import uuid
from abc import ABCMeta
from typing import Iterable

import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.sql.elements import ClauseElement
from sqlalchemy import func

from . import utils, exceptions, aviews, sql_literals


class ConnectionLogger:
    logger = logging.getLogger('common.db')

    def __init__(self, connection):
        self._connection = connection

    def log(self, sql):
        if not self.logger.hasHandlers():
            return
        elif isinstance(sql, str):
            s = sql
        else:
            s = sql.compile(
                dialect=sql_literals.LiteralDialect(),
                compile_kwargs={"literal_binds": True},
            )
        self.logger.debug(s)

    def execute(self, sql, *args, **kwargs):
        self.log(sql)
        return self._connection.execute(sql, *args, **kwargs)

    def scalar(self, sql, *args, **kwargs):
        self.log(sql)
        return self._connection.scalar(sql, *args, **kwargs)


def method_connect_once(func):
    @functools.wraps(func)
    async def wraper(self, *args, **kwargs):
        if kwargs.get('connection') is None:
            async with self.app.db.acquire() as connection:
                kwargs['connection'] = ConnectionLogger(connection)
                return await func(self, *args, **kwargs)
        else:
            return await func(self, *args, **kwargs)
    return wraper


def method_redis_once(arg):
    redis = 'redis'

    def with_arg(func):
        @functools.wraps(func)
        async def wraper(self, *args, **kwargs):
            if kwargs.get(redis) is None:
                async with getattr(self.app, redis).get() as connection:
                    kwargs[redis] = connection
                    return await func(self, *args, **kwargs)
            else:
                return await func(self, *args, **kwargs)
        return wraper

    if not callable(arg):
        redis = arg
        return with_arg

    return with_arg(arg)


class MetaModel(ABCMeta):
    def __new__(mcls, name, bases, namespace):
        cls = ABCMeta.__new__(mcls, name, bases, namespace)
        name = utils.convert_class_name(name)
        cls.models[name] = cls
        return cls


class Model(dict, metaclass=MetaModel):
    models = {}
    app = None
    primary_key = 'id'
    validators = ()
    fields_permanent = ()  # Поля которые необходимо всегда сохранять
    fields_readonly = ()
    fields_list = ()
    fields_one = None
<commit_msg>Use dict to store db and redis.<commit_after>import functools
import itertools
import json
import logging
import uuid
from abc import ABCMeta
from typing import Iterable

import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.sql.elements import ClauseElement
from sqlalchemy import func

from . import utils, exceptions, aviews, sql_literals


class ConnectionLogger:
    logger = logging.getLogger('common.db')

    def __init__(self, connection):
        self._connection = connection

    def log(self, sql):
        if not self.logger.hasHandlers():
            return
        elif isinstance(sql, str):
            s = sql
        else:
            s = sql.compile(
                dialect=sql_literals.LiteralDialect(),
                compile_kwargs={"literal_binds": True},
            )
        self.logger.debug(s)

    def execute(self, sql, *args, **kwargs):
        self.log(sql)
        return self._connection.execute(sql, *args, **kwargs)

    def scalar(self, sql, *args, **kwargs):
        self.log(sql)
        return self._connection.scalar(sql, *args, **kwargs)


def method_connect_once(func):
    @functools.wraps(func)
    async def wraper(self, *args, **kwargs):
        if kwargs.get('connection') is None:
            async with self.app['db'].acquire() as connection:
                kwargs['connection'] = ConnectionLogger(connection)
                return await func(self, *args, **kwargs)
        else:
            return await func(self, *args, **kwargs)
    return wraper


def method_redis_once(arg):
    redis = 'redis'

    def with_arg(func):
        @functools.wraps(func)
        async def wraper(self, *args, **kwargs):
            if kwargs.get(redis) is None:
                async with self.app[redis].get() as connection:
                    kwargs[redis] = connection
                    return await func(self, *args, **kwargs)
            else:
                return await func(self, *args, **kwargs)
        return wraper

    if not callable(arg):
        redis = arg
        return with_arg

    return with_arg(arg)


class MetaModel(ABCMeta):
    def __new__(mcls, name, bases, namespace):
        cls = ABCMeta.__new__(mcls, name, bases, namespace)
        name = utils.convert_class_name(name)
        cls.models[name] = cls
        return cls


class Model(dict, metaclass=MetaModel):
    models = {}
    app = None
    primary_key = 'id'
    validators = ()
    fields_permanent = ()  # Поля которые необходимо всегда сохранять
    fields_readonly = ()
    fields_list = ()
    fields_one = None

{'fraction_non_alphanumeric': 0.10046457607433217, 'fraction_numerical': 0.0, 'mean_word_length': 2.394875164257556, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512637', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1518, 'n_words': 411}