<commit_before>import logging
import datetime
import argparse
import math
import collections
import urllib

from pajbot.models.db import DBManager, Base
from pajbot.models.handler import HandlerManager
from pajbot.managers.redis import RedisManager

from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Boolean
from sqlalchemy.dialects.mysql import BIGINT
from sqlalchemy import orm
from sqlalchemy.orm import relationship
from sqlalchemy import inspect
from sqlalchemy.ext.hybrid import hybrid_property

log = logging.getLogger('pajbot')


def parse_twitch_datetime(datetime_str):
    return datetime.datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%SZ')


class Stream(Base):
    __tablename__ = 'tb_stream'

    id = Column(Integer, primary_key=True)
    title = Column(String(256), nullable=False)
    stream_start = Column(DateTime, nullable=False)
    stream_end = Column(DateTime, nullable=True)
    ended = Column(Boolean, nullable=False, default=False)

    stream_chunks = relationship('StreamChunk',
            backref='stream',
            cascade='save-update, merge, expunge',
            lazy='joined')

    def __init__(self, created_at, **options):
        self.id = None
        self.title = options.get('title', 'NO TITLE')
        self.stream_start = parse_twitch_datetime(created_at)
        self.stream_end = None
        self.ended = False

    @property
    def uptime(self):
        """
        Returns a TimeDelta for how long the stream was online, or is online.
        """

        if self.ended is False:
            return datetime.datetime.now() - self.stream_start
        else:
            return self.stream_end - self.stream_start

class StreamChunk(Base):
    __tablename__ = 'tb_stream_chunk'

    id = Column(Integer, primary_key=True)
    stream_id = Column(Integer, ForeignKey('tb_stream.id'), nullable=False)
    broadcast_id = Column(BIGINT(unsigned=True), nullable=False)
    video_url = Column(String(128), nullable=True)
    video_preview_image_url = Column(String(256), nullable=True)
    chunk_start = Column(DateTime, nullable=False)
    chunk_end = Column(DateTime, nullable=True)

    highlights = relationship('StreamChunkHighlight',
            backref='stream_chunk',
            cascade='save-update, merge, expunge',
            lazy='joined')

    def __init__(self, stream, broadcast_id, created_at, **options):
        self.id = None
        self.stream_id = stream.id
        self.broadcast_id = broadcast_id
        self.video_url = None
        self.video_preview_image_url = None
        self.chunk_start = parse_twitch_datetime(created_at)
        self.chunk_end = None

        self.stream = stream

        self.highlights = []

class StreamChunkHighlight(Base):
    __tablename__ = 'tb_stream_chunk_highlight'

    id = Column(Integer, primary_key=True)
    stream_chunk_id = Column(Integer, ForeignKey('tb_stream_chunk.id'), nullable=False)
    created_by = Column(Integer, ForeignKey('tb_user.id'), nullable=True)
    last_edited_by = Column(Integer, ForeignKey('tb_user.id'), nullable=True)
    created_at = Column(DateTime, nullable=False)
    highlight_offset = Column(Integer, nullable=False)
    description = Column(String(128), nullable=True)
    override_link = Column(String(256), nullable=True)
    thumbnail = Column(Boolean, nullable=True, default=None)
    video_url = None

    created_by_user = relationship('User',
            cascade='save-update, merge, expunge',
            uselist=False)
    last_edited_by_user = relationship('User',
            cascade='save-update, merge, expunge',
            uselist=False)

    DEFAULT_OFFSET = 0

    def __init__(self, stream_chunk, **options):
        self.stream_chunk_id = stream_chunk.id
        self.created_at = datetime.datetime.now()
        self.highlight_offset = options.get('offset', self.DEFAULT_OFFSET)
        self.description = options.get('description', None)
        self.override_link = options.get('override_link', None)
        self.thumbnail = None
        self.created_by = options.get('created_by', None)
        self.last_edited_by = options.get('last_edited_by', None)

        self.stream_chunk = stream_chunk
        self.refresh_video_url()

        stream_chunk.highlights.append(self)

    @orm.reconstructor
    def on_load(self):
        self.refresh_video_url()<commit_msg>fixed some foreign key ambiguity in the StreamChunkHighlight table<commit_after>import logging
import datetime
import argparse
import math
import collections
import urllib

from pajbot.models.db import DBManager, Base
from pajbot.models.handler import HandlerManager
from pajbot.managers.redis import RedisManager

from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Boolean
from sqlalchemy.dialects.mysql import BIGINT
from sqlalchemy import orm
from sqlalchemy.orm import relationship
from sqlalchemy import inspect
from sqlalchemy.ext.hybrid import hybrid_property

log = logging.getLogger('pajbot')


def parse_twitch_datetime(datetime_str):
    return datetime.datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%SZ')


class Stream(Base):
    __tablename__ = 'tb_stream'

    id = Column(Integer, primary_key=True)
    title = Column(String(256), nullable=False)
    stream_start = Column(DateTime, nullable=False)
    stream_end = Column(DateTime, nullable=True)
    ended = Column(Boolean, nullable=False, default=False)

    stream_chunks = relationship('StreamChunk',
            backref='stream',
            cascade='save-update, merge, expunge',
            lazy='joined')

    def __init__(self, created_at, **options):
        self.id = None
        self.title = options.get('title', 'NO TITLE')
        self.stream_start = parse_twitch_datetime(created_at)
        self.stream_end = None
        self.ended = False

    @property
    def uptime(self):
        """
        Returns a TimeDelta for how long the stream was online, or is online.
        """

        if self.ended is False:
            return datetime.datetime.now() - self.stream_start
        else:
            return self.stream_end - self.stream_start

class StreamChunk(Base):
    __tablename__ = 'tb_stream_chunk'

    id = Column(Integer, primary_key=True)
    stream_id = Column(Integer, ForeignKey('tb_stream.id'), nullable=False)
    broadcast_id = Column(BIGINT(unsigned=True), nullable=False)
    video_url = Column(String(128), nullable=True)
    video_preview_image_url = Column(String(256), nullable=True)
    chunk_start = Column(DateTime, nullable=False)
    chunk_end = Column(DateTime, nullable=True)

    highlights = relationship('StreamChunkHighlight',
            backref='stream_chunk',
            cascade='save-update, merge, expunge',
            lazy='joined')

    def __init__(self, stream, broadcast_id, created_at, **options):
        self.id = None
        self.stream_id = stream.id
        self.broadcast_id = broadcast_id
        self.video_url = None
        self.video_preview_image_url = None
        self.chunk_start = parse_twitch_datetime(created_at)
        self.chunk_end = None

        self.stream = stream

        self.highlights = []

class StreamChunkHighlight(Base):
    __tablename__ = 'tb_stream_chunk_highlight'

    id = Column(Integer, primary_key=True)
    stream_chunk_id = Column(Integer, ForeignKey('tb_stream_chunk.id'), nullable=False)
    created_by = Column(Integer, ForeignKey('tb_user.id'), nullable=True)
    last_edited_by = Column(Integer, ForeignKey('tb_user.id'), nullable=True)
    created_at = Column(DateTime, nullable=False)
    highlight_offset = Column(Integer, nullable=False)
    description = Column(String(128), nullable=True)
    override_link = Column(String(256), nullable=True)
    thumbnail = Column(Boolean, nullable=True, default=None)
    video_url = None

    created_by_user = relationship('User',
            foreign_keys='StreamChunkHighlight.created_by',
            cascade='save-update, merge, expunge',
            uselist=False)
    last_edited_by_user = relationship('User',
            foreign_keys='StreamChunkHighlight.last_edited_by',
            cascade='save-update, merge, expunge',
            uselist=False)

    DEFAULT_OFFSET = 0

    def __init__(self, stream_chunk, **options):
        self.stream_chunk_id = stream_chunk.id
        self.created_at = datetime.datetime.now()
        self.highlight_offset = options.get('offset', self.DEFAULT_OFFSET)
        self.description = options.get('description', None)
        self.override_link = options.get('override_link', None)
        self.thumbnail = None
        self.created_by = options.get('created_by', None)
        self.last_edited_by = options.get('last_edited_by', None)

        self.stream_chunk = stream_chunk
        self.refresh_video_url()

        stream_chunk.highlights.append(self)

    @orm.reconstructor
    def on_load(self):
        self.refresh_video_url()
{'fraction_non_alphanumeric': 0.09987627938364639, 'fraction_numerical': 0.003599145203014284, 'mean_word_length': 3.493178372915614, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '320615', 'n_tokens_mistral': 2708, 'n_tokens_neox': 2596, 'n_words': 551}