<commit_before># noqa
# -*- coding: utf-8 -*-

import logging
import os
import shutil
import traceback

from django.conf import settings
from django.contrib.gis.geos import GEOSGeometry
from django.utils import timezone
from django.utils.text import slugify

from celery import shared_task
from raven import Client

from jobs.models import Job, HDXExportRegion
from tasks.models import ExportRun, ExportTask

from feature_selection.feature_selection import FeatureSelection

from utils import map_names_to_formats
from utils.manager import RunManager, Zipper, simplify_max_points

from .email import (
    send_completion_notification,
    send_error_notification,
    send_hdx_completion_notification,
    send_hdx_error_notification,
)

client = Client()

LOG = logging.getLogger(__name__)

class ExportTaskRunner(object):
    def run_task(self, job_uid=None, user=None): # noqa
        LOG.debug('Running Job with id: {0}'.format(job_uid))
        job = Job.objects.get(uid=job_uid)
        if not user:
            user = job.user
        run = ExportRun.objects.create(job=job, user=user, status='SUBMITTED')
        run.save()
        run_uid = str(run.uid)
        LOG.debug('Saved run with id: {0}'.format(run_uid))

        for format_name in job.export_formats:
            ExportTask.objects.create(
                run=run,
                status='PENDING',
                name=format_name
            )
            LOG.debug('Saved task: {0}'.format(format_name))

        run_task_remote.delay(run_uid)
        return run


SIMPLE = '''
waterways:
    types:
        - lines
        - polygons
    select:
        - name
        - waterway
buildings:
    types:
        - lines
        - polygons
    select:
        - name
        - building
    where: building IS NOT NULL
'''


@shared_task(bind=True, ignore_result=True)
def run_task_remote(self, run_uid): # noqa
    run = ExportRun.objects.get(uid=run_uid)
    run.status = 'RUNNING'
    run.started_at = timezone.now()
    run.save()
    LOG.debug('Running ExportRun with id: {0}'.format(run_uid))
    job = run.job

    try:
        stage_dir = os.path.join(settings.EXPORT_STAGING_ROOT, str(run_uid)) + '/'
        if not os.path.exists(stage_dir):
            os.makedirs(stage_dir, 6600)
        run_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT, run_uid)
        if not os.path.exists(run_dir):
            os.makedirs(run_dir)

        aoi = GEOSGeometry(job.the_geom)
        if job.buffer_aoi:
            aoi = aoi.buffer(0.02) # 0.02 degrees is a reasonable amount for an admin 0 boundary
        aoi = simplify_max_points(aoi)
        try:
            feature_selection = job.feature_selection_object
        except Exception:
            feature_selection = FeatureSelection(SIMPLE)

        export_formats = map_names_to_formats(job.export_formats)

        download_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT,run_uid)
        zipper = Zipper(
            slugify(job.name, allow_unicode=True), stage_dir, download_dir, aoi, feature_selection)

        def on_task_start(formatcls):
            LOG.debug('Task Start: {0} for run: {1}'.format(formatcls.name, run_uid))
            if formatcls in export_formats:
                task = ExportTask.objects.get(run__uid=run_uid, name=formatcls.name)
                task.status = 'RUNNING'
                task.started_at = timezone.now()
                task.save()


        def on_task_success(formatcls,results):
            LOG.debug('Task Success: {0} for run: {1}'.format(formatcls.name, run_uid))
            if formatcls in export_formats:
                task = ExportTask.objects.get(run__uid=run_uid, name=formatcls.name)
                zipfiles = zipper.run(results)
                task.filesize_bytes = sum(os.stat(zipfile).st_size for zipfile in zipfiles)
                task.filenames = [os.path.basename(zipfile) for zipfile in zipfiles]
                task.status = 'SUCCESS'
                task.finished_at = timezone.now()
                task.save()

        r = RunManager(<commit_msg>clean up task runner<commit_after># noqa
# -*- coding: utf-8 -*-

import logging
import os
import shutil
import traceback

from django.conf import settings
from django.contrib.gis.geos import GEOSGeometry
from django.utils import timezone
from django.utils.text import slugify

from celery import shared_task
from raven import Client

from jobs.models import Job, HDXExportRegion
from tasks.models import ExportRun, ExportTask

from utils import map_names_to_formats
from utils.manager import RunManager, Zipper, simplify_max_points

from .email import (
    send_completion_notification,
    send_error_notification,
    send_hdx_completion_notification,
    send_hdx_error_notification,
)

client = Client()

LOG = logging.getLogger(__name__)

class ExportTaskRunner(object):
    def run_task(self, job_uid=None, user=None): # noqa
        LOG.debug('Running Job with id: {0}'.format(job_uid))
        job = Job.objects.get(uid=job_uid)
        if not user:
            user = job.user
        run = ExportRun.objects.create(job=job, user=user, status='SUBMITTED')
        run.save()
        run_uid = str(run.uid)
        LOG.debug('Saved run with id: {0}'.format(run_uid))

        for format_name in job.export_formats:
            ExportTask.objects.create(
                run=run,
                status='PENDING',
                name=format_name
            )
            LOG.debug('Saved task: {0}'.format(format_name))

        run_task_remote.delay(run_uid)
        return run

@shared_task(bind=True, ignore_result=True)
def run_task_remote(self, run_uid): # noqa
    run = ExportRun.objects.get(uid=run_uid)
    run.status = 'RUNNING'
    run.started_at = timezone.now()
    run.save()
    LOG.debug('Running ExportRun with id: {0}'.format(run_uid))
    job = run.job

    try:
        stage_dir = os.path.join(settings.EXPORT_STAGING_ROOT, str(run_uid)) + '/'
        if not os.path.exists(stage_dir):
            os.makedirs(stage_dir, 6600)
        run_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT, run_uid)
        if not os.path.exists(run_dir):
            os.makedirs(run_dir)

        aoi = GEOSGeometry(job.the_geom)
        if job.buffer_aoi:
            aoi = aoi.buffer(0.02) # 0.02 degrees is a reasonable amount for an admin 0 boundary
        aoi = simplify_max_points(aoi)
        feature_selection = job.feature_selection_object

        export_formats = map_names_to_formats(job.export_formats)

        download_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT,run_uid)
        zipper = Zipper(
            slugify(job.name, allow_unicode=True), stage_dir, download_dir, aoi, feature_selection)

        def on_task_start(formatcls):
            LOG.debug('Task Start: {0} for run: {1}'.format(formatcls.name, run_uid))
            if formatcls in export_formats:
                task = ExportTask.objects.get(run__uid=run_uid, name=formatcls.name)
                task.status = 'RUNNING'
                task.started_at = timezone.now()
                task.save()


        def on_task_success(formatcls,results):
            LOG.debug('Task Success: {0} for run: {1}'.format(formatcls.name, run_uid))
            if formatcls in export_formats:
                task = ExportTask.objects.get(run__uid=run_uid, name=formatcls.name)
                zipfiles = zipper.run(results)
                task.filesize_bytes = sum(os.stat(zipfile).st_size for zipfile in zipfiles)
                task.filenames = [os.path.basename(zipfile) for zipfile in zipfiles]
                task.status = 'SUCCESS'
                task.finished_at = timezone.now()
                task.save()

        r = RunManager(
{'fraction_non_alphanumeric': 0.09595631257313743, 'fraction_numerical': 0.005200884150305552, 'mean_word_length': 2.5381784728610857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '882051', 'n_tokens_mistral': 2471, 'n_tokens_neox': 2363, 'n_words': 509}