<commit_before>import logging
import time

import django_rq
import os
import requests
import shutil
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand

from osmaxx.conversion import models as conversion_models
from osmaxx.conversion._settings import CONVERSION_SETTINGS
from osmaxx.conversion_api.statuses import FINAL_STATUSES, FINISHED, FAILED

logging.basicConfig()
logger = logging.getLogger(__name__)


class Command(BaseCommand):
    help = 'updates currently active jobs - runs until interrupted'

    def handle(self, *args, **options):
        while True:
            logger.info('handling running jobs')
            self._handle_running_jobs()
            logger.info('handling failed jobs')
            self._handle_failed_jobs()
            time.sleep(CONVERSION_SETTINGS['result_harvest_interval_seconds'])

    def _handle_failed_jobs(self):
        failed_queue = django_rq.get_failed_queue()
        for job_id in failed_queue.job_ids:
            self._update_job(job_id=job_id, queue=failed_queue)

    def _handle_running_jobs(self):
        queue = django_rq.get_queue()
        active_jobs = conversion_models.Job.objects.exclude(status__in=FINAL_STATUSES)\
            .values_list('rq_job_id', flat=True)
        for job_id in active_jobs:
            self._update_job(job_id=job_id, queue=queue)

    def _update_job(self, job_id, queue):
        if job_id is None:
            logger.error("job_id is None, None is not a valid id!")
            return

        job = queue.fetch_job(job_id)

        try:
            conversion_job = conversion_models.Job.objects.get(rq_job_id=job_id)
        except ObjectDoesNotExist as e:
            logger.exception(e)
            return

        if job is None:  # already processed by someone else
            conversion_job.refresh_from_db()
            if conversion_job.status not in FINAL_STATUSES:
                logger.warning("job {} not found in queue but status is {} on database.".format(
                    job_id, conversion_job.status
                ))
                conversion_job.status = FAILED<commit_msg>log warning as error<commit_after>import logging
import time

import django_rq
import os
import requests
import shutil
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand

from osmaxx.conversion import models as conversion_models
from osmaxx.conversion._settings import CONVERSION_SETTINGS
from osmaxx.conversion_api.statuses import FINAL_STATUSES, FINISHED, FAILED

logging.basicConfig()
logger = logging.getLogger(__name__)


class Command(BaseCommand):
    help = 'updates currently active jobs - runs until interrupted'

    def handle(self, *args, **options):
        while True:
            logger.info('handling running jobs')
            self._handle_running_jobs()
            logger.info('handling failed jobs')
            self._handle_failed_jobs()
            time.sleep(CONVERSION_SETTINGS['result_harvest_interval_seconds'])

    def _handle_failed_jobs(self):
        failed_queue = django_rq.get_failed_queue()
        for job_id in failed_queue.job_ids:
            self._update_job(job_id=job_id, queue=failed_queue)

    def _handle_running_jobs(self):
        queue = django_rq.get_queue()
        active_jobs = conversion_models.Job.objects.exclude(status__in=FINAL_STATUSES)\
            .values_list('rq_job_id', flat=True)
        for job_id in active_jobs:
            self._update_job(job_id=job_id, queue=queue)

    def _update_job(self, job_id, queue):
        if job_id is None:
            logger.error("job_id is None, None is not a valid id!")
            return

        job = queue.fetch_job(job_id)

        try:
            conversion_job = conversion_models.Job.objects.get(rq_job_id=job_id)
        except ObjectDoesNotExist as e:
            logger.exception(e)
            return

        if job is None:  # already processed by someone else
            conversion_job.refresh_from_db()
            if conversion_job.status not in FINAL_STATUSES:
                logger.error("job {} not found in queue but status is {} on database.".format(
                    job_id, conversion_job.status
                ))
                conversion_job.status = FAILED
{'fraction_non_alphanumeric': 0.07281886878864209, 'fraction_numerical': 0.0, 'mean_word_length': 2.7982608695652176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5933387', 'n_tokens_mistral': 1264, 'n_tokens_neox': 1205, 'n_words': 302}