<commit_before>from datetime import datetime, timedelta
from sqlalchemy import func
from sqlalchemy.dialects.postgresql import JSONB

from aleph.core import db
from aleph.model.collection import Collection


class CrawlerState(db.Model):
    """Report the state of a file being processed."""

    TIMEOUT = timedelta(minutes=60)

    STATUS_OK = 'ok'
    STATUS_FAIL = 'fail'

    id = db.Column(db.BigInteger, primary_key=True)
    crawler_id = db.Column(db.Unicode(), index=True)
    crawler_run = db.Column(db.Unicode(), nullable=True)
    content_hash = db.Column(db.Unicode(65), nullable=True)
    foreign_id = db.Column(db.Unicode, nullable=True)
    status = db.Column(db.Unicode(10), nullable=False)
    error_type = db.Column(db.Unicode(), nullable=True)
    error_message = db.Column(db.Unicode(), nullable=True)
    error_details = db.Column(db.Unicode(), nullable=True)
    meta = db.Column(JSONB)
    collection_id = db.Column(db.Integer(), db.ForeignKey('collection.id'), index=True)
    collection = db.relationship(Collection, backref=db.backref('crawl_states', cascade='all, delete-orphan'))  # noqa
    created_at = db.Column(db.DateTime, default=datetime.utcnow)

    @classmethod
    def _from_meta(cls, meta, collection_id):
        obj = cls()
        obj.collection_id = collection_id
        obj.crawler_id = meta.crawler
        obj.crawler_run = meta.crawler_run
        obj.foreign_id = meta.foreign_id
        obj.content_hash = meta.content_hash
        obj.meta = meta.to_attr_dict()
        db.session.add(obj)
        return obj

    @classmethod
    def store_stub(cls, collection_id, crawler_id, crawler_run):
        obj = cls()
        obj.collection_id = collection_id
        obj.crawler_id = crawler_id
        obj.crawler_run = crawler_run
        obj.error_type = 'init'
        obj.status = cls.STATUS_OK
        db.session.add(obj)
        return obj

    @classmethod
    def store_ok(cls, meta, collection_id):
        obj = cls._from_meta(meta, collection_id)
        obj.status = cls.STATUS_OK
        return obj

    @classmethod
    def store_fail(cls, meta, collection_id, error_type=None,
                   error_message=None, error_details=None):
        obj = cls._from_meta(meta, collection_id)
        obj.status = cls.STATUS_FAIL
        obj.error_type = error_type
        obj.error_message = error_message
        obj.error_details = error_details
        return obj

    @classmethod
    def crawler_last_run(cls, crawler_id):
        q = db.session.query(cls.crawler_run, cls.created_at)
        q = q.filter(cls.crawler_id == crawler_id)
        q = q.order_by(cls.created_at.desc())
        q = q.limit(1)
        res = q.first()
        if res is None:
            return None, None
        return (res.crawler_run, res.created_at)

    @classmethod
    def crawler_stats(cls, crawler_id):
        stats = {}
        last_run_id, last_run_time = cls.crawler_last_run(crawler_id)

        # Check if the crawler was active very recently, if so, don't
        # allow the user to execute a new run right now.
        timeout = (datetime.utcnow() - CrawlerState.TIMEOUT)
        stats['running'] = last_run_time > timeout if last_run_time else False

        q = db.session.query(func.count(func.distinct(cls.foreign_id)))
        q = q.filter(cls.crawler_id == crawler_id)
        for section in ['last', 'all']:
            data = {}
            sq = q<commit_msg>stupid aggregation change<commit_after>from datetime import datetime, timedelta
from sqlalchemy import func
from sqlalchemy.dialects.postgresql import JSONB

from aleph.core import db
from aleph.model.collection import Collection


class CrawlerState(db.Model):
    """Report the state of a file being processed."""

    TIMEOUT = timedelta(minutes=60)

    STATUS_OK = 'ok'
    STATUS_FAIL = 'fail'

    id = db.Column(db.BigInteger, primary_key=True)
    crawler_id = db.Column(db.Unicode(), index=True)
    crawler_run = db.Column(db.Unicode(), nullable=True)
    content_hash = db.Column(db.Unicode(65), nullable=True)
    foreign_id = db.Column(db.Unicode, nullable=True)
    status = db.Column(db.Unicode(10), nullable=False)
    error_type = db.Column(db.Unicode(), nullable=True)
    error_message = db.Column(db.Unicode(), nullable=True)
    error_details = db.Column(db.Unicode(), nullable=True)
    meta = db.Column(JSONB)
    collection_id = db.Column(db.Integer(), db.ForeignKey('collection.id'), index=True)
    collection = db.relationship(Collection, backref=db.backref('crawl_states', cascade='all, delete-orphan'))  # noqa
    created_at = db.Column(db.DateTime, default=datetime.utcnow)

    @classmethod
    def _from_meta(cls, meta, collection_id):
        obj = cls()
        obj.collection_id = collection_id
        obj.crawler_id = meta.crawler
        obj.crawler_run = meta.crawler_run
        obj.foreign_id = meta.foreign_id
        obj.content_hash = meta.content_hash
        obj.meta = meta.to_attr_dict()
        db.session.add(obj)
        return obj

    @classmethod
    def store_stub(cls, collection_id, crawler_id, crawler_run):
        obj = cls()
        obj.collection_id = collection_id
        obj.crawler_id = crawler_id
        obj.crawler_run = crawler_run
        obj.error_type = 'init'
        obj.status = cls.STATUS_OK
        db.session.add(obj)
        return obj

    @classmethod
    def store_ok(cls, meta, collection_id):
        obj = cls._from_meta(meta, collection_id)
        obj.status = cls.STATUS_OK
        return obj

    @classmethod
    def store_fail(cls, meta, collection_id, error_type=None,
                   error_message=None, error_details=None):
        obj = cls._from_meta(meta, collection_id)
        obj.status = cls.STATUS_FAIL
        obj.error_type = error_type
        obj.error_message = error_message
        obj.error_details = error_details
        return obj

    @classmethod
    def crawler_last_run(cls, crawler_id):
        q = db.session.query(cls.crawler_run, cls.created_at)
        q = q.filter(cls.crawler_id == crawler_id)
        q = q.order_by(cls.created_at.desc())
        q = q.limit(1)
        res = q.first()
        if res is None:
            return None, None
        return (res.crawler_run, res.created_at)

    @classmethod
    def crawler_stats(cls, crawler_id):
        stats = {}
        last_run_id, last_run_time = cls.crawler_last_run(crawler_id)

        # Check if the crawler was active very recently, if so, don't
        # allow the user to execute a new run right now.
        timeout = (datetime.utcnow() - CrawlerState.TIMEOUT)
        stats['running'] = last_run_time > timeout if last_run_time else False

        q = db.session.query(func.count(func.distinct(cls.id)))
        q = q.filter(cls.crawler_id == crawler_id)
        for section in ['last', 'all']:
            data = {}
            sq = q
{'fraction_non_alphanumeric': 0.10469524642752989, 'fraction_numerical': 0.002041411490230388, 'mean_word_length': 2.7915975677169707, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7529541', 'n_tokens_mistral': 2260, 'n_tokens_neox': 2237, 'n_words': 467}