<commit_before>import logging
from datetime import datetime
from time import time

from biothings.utils.common import merge, timesofar

# NO CONCURRENT
# TASK SUPPORT YET

_map = {
    # "pre": PreSnapshotState,
    # "snapshot": MainSnapshotState,
    # "post": PostSnapshotState
}

def dispatch(step):
    return _map[step]

def audit(src_build, logger=None):
    if not logger:
        logger = logging.getLogger(__name__)

    for build in src_build.find():
        for num, job in enumerate(build.get("jobs", [])):

            if job.get("status") == "in progress":
                job["status"] = "canceled"

                msg = "<Job #%d in <Build '%s'> cancelled>"
                logger.warning(msg, num, build["_id"])

        src_build.replace_one({"_id": build["_id"]}, build)


class _TaskState():
    name = NotImplemented  # string representation of the step
    step = NotImplemented  # job registration display notation
    func = NotImplemented  # method name of the step to run
    regx = False  # ready to update the build doc

    def __init__(self, col, _id):
        self._col = col
        self._id = _id

    @classmethod
    def __init_subclass__(cls):
        _map[cls.name] = cls

    def started(self, **extras):
        timestamp = datetime.now().astimezone()
        self._col.update(
            {"_id": self._id},
            {"$push": {
                "jobs": {
                    "step": self.step,
                    "status": "in progress",
                    "step_started_at": timestamp,
                    **extras
                }
            }}
        )

    def _finished(self, _doc, _job):
        doc = self._col.find_one({'_id': self._id})
        job = doc["jobs"][-1]

        t0 = job["step_started_at"].timestamp()<commit_msg>Fix snapshot build time. Issue #248<commit_after>import logging
from datetime import datetime, timezone
from time import time

from biothings.utils.common import merge, timesofar

# NO CONCURRENT
# TASK SUPPORT YET

_map = {
    # "pre": PreSnapshotState,
    # "snapshot": MainSnapshotState,
    # "post": PostSnapshotState
}

def dispatch(step):
    return _map[step]

def audit(src_build, logger=None):
    if not logger:
        logger = logging.getLogger(__name__)

    for build in src_build.find():
        for num, job in enumerate(build.get("jobs", [])):

            if job.get("status") == "in progress":
                job["status"] = "canceled"

                msg = "<Job #%d in <Build '%s'> cancelled>"
                logger.warning(msg, num, build["_id"])

        src_build.replace_one({"_id": build["_id"]}, build)


class _TaskState():
    name = NotImplemented  # string representation of the step
    step = NotImplemented  # job registration display notation
    func = NotImplemented  # method name of the step to run
    regx = False  # ready to update the build doc

    def __init__(self, col, _id):
        self._col = col
        self._id = _id

    @classmethod
    def __init_subclass__(cls):
        _map[cls.name] = cls

    def started(self, **extras):
        timestamp = datetime.now().astimezone()
        self._col.update(
            {"_id": self._id},
            {"$push": {
                "jobs": {
                    "step": self.step,
                    "status": "in progress",
                    "step_started_at": timestamp,
                    **extras
                }
            }}
        )

    def _finished(self, _doc, _job):
        doc = self._col.find_one({'_id': self._id})
        job = doc["jobs"][-1]
        t0 = job["step_started_at"]

        # New pymongo lib version changed the default configuration value of `tz_aware` to `False` instead of `True`.
        # Because of that the datetime value read from mongodb was not including the timezone anymore.
        # The solution is forcing the datetime value to include the UTC timezone when it is missing.
        # Reference: https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html?highlight=datetime#tz-aware-defaults-to-false
        if not t0.tzinfo:
            t0 = t0.replace(tzinfo=timezone.utc)
        t0 = t0.timestamp()

{'fraction_non_alphanumeric': 0.11708022163334136, 'fraction_numerical': 0.0031317754757889667, 'mean_word_length': 2.171886936592819, 'pattern_counts': {'":': 22, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1878166', 'n_tokens_mistral': 1252, 'n_tokens_neox': 1201, 'n_words': 343}