<commit_before># -*- coding: UTF-8 -*-
from datetime import datetime
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()


class LiveTVSite(Base):
    """ 直播站点 """
    __tablename__ = 'livetv_site'

    id = Column(Integer, primary_key=True)
    channels = relationship('LiveTVChannel', backref='site', lazy='dynamic')
    rooms = relationship('LiveTVRoom', backref='site', lazy='dynamic')

    code = Column(String(32), unique=True, index=True, doc='代码')
    name = Column(String(64), doc='名称')
    url = Column(String(1024), unique=True, index=True, doc='官网地址')
    image = Column(String(1024), doc='图片')
    show_seq = Column(Integer, index=True, doc='排序')
    description = Column(String(512), doc='站点描述')
    valid = Column(Boolean, default=True, doc='是否有效')


class LiveTVChannel(Base):
    """ 频道 """
    __tablename__ = 'livetv_channel'

    id = Column(Integer, primary_key=True)
    site_id = Column(Integer, ForeignKey('livetv_site.id'))
    rooms = relationship('LiveTVRoom', backref='channel', lazy='dynamic')

    office_id = Column(String(32), index=True, doc='官方ID')
    short = Column(String(64), index=True, doc='简称')
    name = Column(String(128), index=True, doc='名称')
    url = Column(String(1024), unique=True, doc='官网地址')
    image = Column(String, doc='图片')
    total = Column(Integer, default=0, index=True, doc='房间数')
    valid = Column(Boolean, default=True, doc='是否有效')
    crawl_date = Column(DateTime, doc='最近一次扫描时间')

    def from_item(self, item):
        self.office_id = item.get('office_id', '')
        self.short = item['short']
        self.name = item['name']
        self.url = item['url']
        self.image = item.get('image', '')
        self.valid = True
        self.crawl_date = datetime.utcnow()


class LiveTVRoom(Base):
    """ 房间 """
    __tablename__ = 'livetv_room'
    id = Column(Integer, primary_key=True)
    channel_id = Column(Integer, ForeignKey('livetv_channel.id'))
    site_id = Column(Integer, ForeignKey('livetv_site.id'))

    office_id = Column(String(32), index=True, doc='官方ID')
    name = Column(String(258), index=True, doc='名称')
    url = Column(String(1024), index=True, doc='访问URL')
    image = Column(String(1024), doc='图片')
    host = Column(String(128), doc='主持')
    online = Column(Integer, default=0, index=True, doc='观众数')
    opened = Column(Boolean, default=True, doc='是否正在直播')
    crawl_date = Column(DateTime, doc='最近一次扫描时间')

    def from_item(self, item):
        self.office_id = item['office_id']
        self.name = item['name']
        self.url = item['url']
        self.image = item['image']
        self.host = item['host']
        self.online = item['online']
        self.opened = True
        self.crawl_date = datetime.utcnow()
<commit_msg>add crawler table model index.<commit_after># -*- coding: UTF-8 -*-
from datetime import datetime
from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()


class LiveTVSite(Base):
    """ 直播站点 """
    __tablename__ = 'livetv_site'

    id = Column(Integer, primary_key=True)
    channels = relationship('LiveTVChannel', backref='site', lazy='dynamic')
    rooms = relationship('LiveTVRoom', backref='site', lazy='dynamic')

    code = Column(String(32), unique=True, index=True, doc='代码')
    name = Column(String(64), doc='名称')
    url = Column(String(1024), unique=True, index=True, doc='官网地址')
    image = Column(String(1024), doc='图片')
    show_seq = Column(Integer, index=True, doc='排序')
    description = Column(String(512), doc='站点描述')
    valid = Column(Boolean, default=True, doc='是否有效')


class LiveTVChannel(Base):
    """ 频道 """
    __tablename__ = 'livetv_channel'

    id = Column(Integer, primary_key=True)
    site_id = Column(Integer, ForeignKey('livetv_site.id'))
    rooms = relationship('LiveTVRoom', backref='channel', lazy='dynamic')

    office_id = Column(String(32), index=True, doc='官方ID')
    short = Column(String(64), index=True, doc='简称')
    name = Column(String(128), index=True, doc='名称')
    url = Column(String(1024), unique=True, doc='官网地址')
    image = Column(String, doc='图片')
    total = Column(Integer, default=0, index=True, doc='房间数')
    valid = Column(Boolean, default=True, doc='是否有效')
    crawl_date = Column(DateTime, doc='最近一次扫描时间')

    def from_item(self, item):
        self.office_id = item.get('office_id', '')
        self.short = item['short']
        self.name = item['name']
        self.url = item['url']
        self.image = item.get('image', '')
        self.valid = True
        self.crawl_date = datetime.utcnow()


class LiveTVRoom(Base):
    """ 房间 """
    __tablename__ = 'livetv_room'
    id = Column(Integer, primary_key=True)
    channel_id = Column(Integer, ForeignKey('livetv_channel.id'))
    site_id = Column(Integer, ForeignKey('livetv_site.id'))

    office_id = Column(String(32), index=True, doc='官方ID')
    name = Column(String(258), index=True, doc='名称')
    url = Column(String(1024), index=True, doc='访问URL')
    image = Column(String(1024), doc='图片')
    host = Column(String(128), doc='主持')
    online = Column(Integer, default=0, index=True, doc='观众数')
    opened = Column(Boolean, default=True, index=True, doc='是否正在直播')
    crawl_date = Column(DateTime, index=True, doc='最近一次扫描时间')

    def from_item(self, item):
        self.office_id = item['office_id']
        self.name = item['name']
        self.url = item['url']
        self.image = item['image']
        self.host = item['host']
        self.online = item['online']
        self.opened = True
        self.crawl_date = datetime.utcnow()

{'fraction_non_alphanumeric': 0.16015826595561672, 'fraction_numerical': 0.015482539136418373, 'mean_word_length': 3.9777397260273974, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4976892', 'n_tokens_mistral': 2046, 'n_tokens_neox': 2027, 'n_words': 396}