<commit_before>"""
This module contains
"""
import time
from datetime import timedelta, datetime
from django.core.cache import cache
from celery import task, group
from treeherder.model.derived import RefDataManager, JobsModel
from treeherder.model.models import ReferenceDataSignatures
from .buildapi import (RunningJobsProcess,
                       PendingJobsProcess,
                       Builds4hJobsProcess,
                       Builds4hAnalyzer)
from .bugzilla import BzApiBugProcess
from .tbpl import OrangeFactorBugRequest, TbplBugRequest
from .pushlog import HgPushlogProcess


@task(name='fetch-buildapi-pending')
def fetch_buildapi_pending():
    """
    Fetches the buildapi pending jobs api and load them to
    the objectstore ingestion endpoint
    """
    PendingJobsProcess().run()


@task(name='fetch-buildapi-running')
def fetch_buildapi_running():
    """
    Fetches the buildapi running jobs api and load them to
    the objectstore ingestion endpoint
    """
    RunningJobsProcess().run()


@task(name='fetch-buildapi-build4h')
def fetch_buildapi_build4h():
    """
    Fetches the buildapi running jobs api and load them to
    the objectstore ingestion endpoint
    """
    Builds4hJobsProcess().run()


@task(name='fetch-push-logs')
def fetch_push_logs():
    """
    Run several fetch_hg_push_log subtasks, one per repository
    """
    rdm = RefDataManager()
    try:
        repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
        # create a group of subtasks and apply them
        g = group(fetch_hg_push_log.si(repo['name'], repo['url'])
                            for repo in repos if repo['dvcs_type'] == 'hg')
        g()
    finally:
        rdm.disconnect()
    # TODO: implement the git pushlog retrieval


@task(name='fetch-hg-push-logs')
def fetch_hg_push_log(repo_name, repo_url):
    """
    Run a HgPushlog etl process
    """
    process = HgPushlogProcess()
    process.run(repo_url+'/json-pushes/?full=1', repo_name)


@task(name='fetch-bugs')
def fetch_bugs():
    """
    Run a BzApiBug process
    """
    process = BzApiBugProcess()
    process.run()


@task(name='run-builds4h-analyzer')
def run_builds4h_analyzer():
    """
    Run a Builds4h Analysis process
    """
    process = Builds4hAnalyzer()
    process.run()


@task(name="submit-star-comment")
def submit_star_comment(project, job_id, bug_id, submit_timestamp, who):
    """
    Send a post request to tbpl's starcomment.php containing a bug association.
    starcomment.php proxies then the request to orange factor
    """
    req = OrangeFactorBugRequest(project, job_id, bug_id, submit_timestamp, who)
    req.generate_request_body()
    req.send_request()


<commit_msg>set a hard timeout on etl celery tasks<commit_after>"""
This module contains
"""
import time
from datetime import timedelta, datetime
from django.core.cache import cache
from celery import task, group
from treeherder.model.derived import RefDataManager, JobsModel
from treeherder.model.models import ReferenceDataSignatures
from .buildapi import (RunningJobsProcess,
                       PendingJobsProcess,
                       Builds4hJobsProcess,
                       Builds4hAnalyzer)
from .bugzilla import BzApiBugProcess
from .tbpl import OrangeFactorBugRequest, TbplBugRequest
from .pushlog import HgPushlogProcess


@task(name='fetch-buildapi-pending', time_limit=60)
def fetch_buildapi_pending():
    """
    Fetches the buildapi pending jobs api and load them to
    the objectstore ingestion endpoint
    """
    PendingJobsProcess().run()


@task(name='fetch-buildapi-running', time_limit=60)
def fetch_buildapi_running():
    """
    Fetches the buildapi running jobs api and load them to
    the objectstore ingestion endpoint
    """
    RunningJobsProcess().run()


@task(name='fetch-buildapi-build4h', time_limit=120)
def fetch_buildapi_build4h():
    """
    Fetches the buildapi running jobs api and load them to
    the objectstore ingestion endpoint
    """
    Builds4hJobsProcess().run()


@task(name='fetch-push-logs')
def fetch_push_logs():
    """
    Run several fetch_hg_push_log subtasks, one per repository
    """
    rdm = RefDataManager()
    try:
        repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
        # create a group of subtasks and apply them
        g = group(fetch_hg_push_log.si(repo['name'], repo['url'])
                            for repo in repos if repo['dvcs_type'] == 'hg')
        g()
    finally:
        rdm.disconnect()


@task(name='fetch-hg-push-logs', time_limit=30)
def fetch_hg_push_log(repo_name, repo_url):
    """
    Run a HgPushlog etl process
    """
    process = HgPushlogProcess()
    process.run(repo_url+'/json-pushes/?full=1', repo_name)


@task(name='fetch-bugs', time_limit=60*5)
def fetch_bugs():
    """
    Run a BzApiBug process
    """
    process = BzApiBugProcess()
    process.run()


@task(name='run-builds4h-analyzer')
def run_builds4h_analyzer():
    """
    Run a Builds4h Analysis process
    """
    process = Builds4hAnalyzer()
    process.run()


@task(name="submit-star-comment")
def submit_star_comment(project, job_id, bug_id, submit_timestamp, who):
    """
    Send a post request to tbpl's starcomment.php containing a bug association.
    starcomment.php proxies then the request to orange factor
    """
    req = OrangeFactorBugRequest(project, job_id, bug_id, submit_timestamp, who)
    req.generate_request_body()
    req.send_request()



{'fraction_non_alphanumeric': 0.10264357338195078, 'fraction_numerical': 0.005834092980856883, 'mean_word_length': 3.4385113268608416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5796706', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1652, 'n_words': 438}