<commit_before>import datetime
import json
import logging
from contextlib import contextmanager

import sqlalchemy
from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import Integer
from sqlalchemy import String

from pajbot.exc import FailedCommand
from pajbot.managers.db import Base
from pajbot.managers.db import DBManager
from pajbot.managers.redis import RedisManager
from pajbot.managers.schedule import ScheduleManager
from pajbot.managers.time import TimeManager
from pajbot.streamhelper import StreamHelper
from pajbot.tbutil import time_method  # NOQA

log = logging.getLogger(__name__)


class User(Base):
    __tablename__ = 'tb_user'

    id = Column(Integer, primary_key=True)
    username = Column(String(32), nullable=False, index=True, unique=True)
    username_raw = Column(String(32))
    level = Column(Integer, nullable=False, default=100)
    points = Column(Integer, nullable=False, default=0, index=True)
    subscriber = Column(Boolean, nullable=False, default=False)
    minutes_in_chat_online = Column(Integer, nullable=False, default=0)
    minutes_in_chat_offline = Column(Integer, nullable=False, default=0)

    def __init__(self, username):
        self.id = None
        self.username = username.lower()
        self.username_raw = username
        self.level = 100
        self.points = 0
        self.subscriber = False
        self.minutes_in_chat_online = 0
        self.minutes_in_chat_offline = 0

        self.quest_progress = {}
        self.debts = []

        self.timed_out = False

    @classmethod
    def test_user(cls, username):
        user = cls()

        user.id = 123
        user.username = username.lower()
        user.username_raw = username
        user.level = 2000
        user.subscriber = True
        user.points = 1234
        user.minutes_in_chat_online = 5
        user.minutes_in_chat_offline = 15

        return user


class NoCacheHit(Exception):
    pass


class UserSQLCache:
    cache = {}

    def init():
        ScheduleManager.execute_every(30 * 60, UserSQLCache._clear_cache)

    def _clear_cache():
        UserSQLCache.cache = {}

    def save(user):
        UserSQLCache.cache[user.username] = {
                'id': user.id,
                'level': user.level,
                'subscriber': user.subscriber,
                }

    def get(username, value):
        if username not in UserSQLCache.cache:
            raise NoCacheHit('User not in cache')

        if value not in UserSQLCache.cache[username]:
            raise NoCacheHit('Value not in cache')

        # log.debug('Returning {}:{} from cache'.format(username, value))
        return UserSQLCache.cache[username][value]


class UserSQL:
    def __init__(self, username, db_session, user_model=None):
        self.username = username
        self.user_model = user_model
        self.model_loaded = user_model is not None
        self.shared_db_session = db_session

    def select_or_create(db_session, username):
        user = db_session.query(User).filter_by(username=username).one_or_none()
        if user is None:
            user = User(username)
            db_session.add(user)
        return user

    # @time_method
    def sql_load(self):
        if self.model_loaded:
            return

        self.model_loaded = True

        log.debug('[UserSQL] Loading user model for {}'.format(self.username))
        # from pajbot.tbutil import print_traceback
        # print_traceback()

        if self.shared_db_session:
            user = UserSQL.select_or_create(self.shared_db_session, self.username)
        else:
            with DBManager.create_session_scope(expire_on_commit=False) as db_session:
                user = UserSQL.select_or_create(db_session, self.username)
                db_session.expunge(user)

        self.user_model = user

    def sql_save(self, save_to_db=True):
        if not self.model_loaded:
            return

        if save_to_db and not self.shared_db_session:
            with DBManager.create_session_scope(expire_on_commit=False) as db_session:
                db_session.add(self.user_model)

        UserSQLCache.save(self.user_model)
<commit_msg>added some debug info to investigate crashes<commit_after>import datetime
import json
import logging
from contextlib import contextmanager

import sqlalchemy
from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import Integer
from sqlalchemy import String

from pajbot.exc import FailedCommand
from pajbot.managers.db import Base
from pajbot.managers.db import DBManager
from pajbot.managers.redis import RedisManager
from pajbot.managers.schedule import ScheduleManager
from pajbot.managers.time import TimeManager
from pajbot.streamhelper import StreamHelper
from pajbot.tbutil import time_method  # NOQA

log = logging.getLogger(__name__)


class User(Base):
    __tablename__ = 'tb_user'

    id = Column(Integer, primary_key=True)
    username = Column(String(32), nullable=False, index=True, unique=True)
    username_raw = Column(String(32))
    level = Column(Integer, nullable=False, default=100)
    points = Column(Integer, nullable=False, default=0, index=True)
    subscriber = Column(Boolean, nullable=False, default=False)
    minutes_in_chat_online = Column(Integer, nullable=False, default=0)
    minutes_in_chat_offline = Column(Integer, nullable=False, default=0)

    def __init__(self, username):
        self.id = None
        self.username = username.lower()
        self.username_raw = username
        self.level = 100
        self.points = 0
        self.subscriber = False
        self.minutes_in_chat_online = 0
        self.minutes_in_chat_offline = 0

        self.quest_progress = {}
        self.debts = []

        self.timed_out = False

    @classmethod
    def test_user(cls, username):
        user = cls()

        user.id = 123
        user.username = username.lower()
        user.username_raw = username
        user.level = 2000
        user.subscriber = True
        user.points = 1234
        user.minutes_in_chat_online = 5
        user.minutes_in_chat_offline = 15

        return user


class NoCacheHit(Exception):
    pass


class UserSQLCache:
    cache = {}

    def init():
        ScheduleManager.execute_every(30 * 60, UserSQLCache._clear_cache)

    def _clear_cache():
        UserSQLCache.cache = {}

    def save(user):
        UserSQLCache.cache[user.username] = {
                'id': user.id,
                'level': user.level,
                'subscriber': user.subscriber,
                }

    def get(username, value):
        if username not in UserSQLCache.cache:
            raise NoCacheHit('User not in cache')

        if value not in UserSQLCache.cache[username]:
            raise NoCacheHit('Value not in cache')

        # log.debug('Returning {}:{} from cache'.format(username, value))
        return UserSQLCache.cache[username][value]


class UserSQL:
    def __init__(self, username, db_session, user_model=None):
        self.username = username
        self.user_model = user_model
        self.model_loaded = user_model is not None
        self.shared_db_session = db_session

    def select_or_create(db_session, username):
        user = db_session.query(User).filter_by(username=username).one_or_none()
        if user is None:
            user = User(username)
            db_session.add(user)
        return user

    # @time_method
    def sql_load(self):
        if self.model_loaded:
            return

        self.model_loaded = True

        log.debug('[UserSQL] Loading user model for {}'.format(self.username))
        # from pajbot.tbutil import print_traceback
        # print_traceback()

        if self.shared_db_session:
            user = UserSQL.select_or_create(self.shared_db_session, self.username)
        else:
            with DBManager.create_session_scope(expire_on_commit=False) as db_session:
                user = UserSQL.select_or_create(db_session, self.username)
                db_session.expunge(user)

        self.user_model = user

    def sql_save(self, save_to_db=True):
        if not self.model_loaded:
            return

        try:
            if save_to_db and not self.shared_db_session:
                with DBManager.create_session_scope(expire_on_commit=False) as db_session:
                    # log.debug('Calling db_session.add on {}'.format(self.user_model))
                    db_session.add(self.user_model)

            UserSQLCache.save(self.user_model)
        except:
            log.exception('Caught exception in sql_save while saving {}'.format(self.user_model))

{'fraction_non_alphanumeric': 0.08279220779220779, 'fraction_numerical': 0.007884972170686457, 'mean_word_length': 2.667091836734694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2933065', 'n_tokens_mistral': 2620, 'n_tokens_neox': 2478, 'n_words': 583}