<commit_before>import sys
from StringIO import StringIO
from celery import task
from django.db import transaction
from django.conf import settings
from django.core.mail import mail_admins
from odk_viewer.models import Export
from utils.export_tools import generate_export,\
    generate_attachments_zip_export, generate_kml_export
from utils.logger_tools import mongo_sync_status, report_exception
from pandas_mongo_bridge import NoRecordsFoundError


def create_async_export(xform, export_type, query, force_xlsx, options=None):
    username = xform.user.username
    id_string = xform.id_string

    @transaction.commit_on_success
    def _create_export(xform, export_type):
        return Export.objects.create(xform=xform, export_type=export_type)

    export = _create_export(xform, export_type)
    result = None
    arguments = {
        'username': username,
        'id_string': id_string,
        'export_id': export.id,
        'query': query,
    }
    if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT,
                       Export.CSV_EXPORT, Export.CSV_ZIP_EXPORT]:
        if options and "group_delimiter" in options:
            arguments["group_delimiter"] = options["group_delimiter"]
        if options and "split_select_multiples" in options:
            arguments["split_select_multiples"] =\
                options["split_select_multiples"]

        # start async export
        if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT]:
            result = create_xls_export.apply_async((), arguments, countdown=10)
        elif export_type == Export.CSV_EXPORT:
            result = create_csv_export.apply_async(
                (), arguments, countdown=10)
        elif export_type == Export.CSV_ZIP_EXPORT:
            result = create_csv_zip_export.apply_async(
                (), arguments, countdown=10)
        else:
            raise Export.ExportTypeError
    elif export_type == Export.ZIP_EXPORT:
        # start async export
        result = create_zip_export.apply_async(
            (), arguments, countdown=10)
    elif export_type == Export.KML_EXPORT:
        # start async export
        result = create_kml_export.apply_async(
            (), arguments, countdown=10)
    else:
        raise Export.ExportTypeError
    if result:
        # when celery is running eager, the export has been generated by the
        # time we get here so lets retrieve the export object a fresh before we
        # save
        if settings.CELERY_ALWAYS_EAGER:
            export = Export.objects.get(id=export.id)
        export.task_id = result.task_id
        export.save()
        return export, result
    return None


@task()
def create_xls_export(username, id_string, export_id, query=None,
                      force_xlsx=True, group_delimiter='/',
                      split_select_multiples=True):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    ext = 'xls' if not force_xlsx else 'xlsx'

    export = Export.objects.get(id=export_id)
    # though export is not available when for has 0 submissions, we
    # catch this since it potentially stops celery
    try:
        gen_export = generate_export(
            Export.XLS_EXPORT, ext, username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        report_exception("XLS Export Exception", e)
        #raise for now to let celery know we failed
        # - doesnt seem to break celery`
        raise
    else:
        return gen_export.id


@task()
def create_csv_export(username, id_string, export_id, query=None,
                      group_delimiter='/', split_select_multiples=True):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(
            Export.CSV_EXPORT, 'csv', username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        report_exception("CSV Export Exception", e)
        raise
    else:
        return gen_export.id


@task()
def create_kml_export(username, id_string, export_id, query=None):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state

    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_kml_export(
            Export.KML_EXPORT, 'kml', username, id_string, export_id, query)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        report_exception("KML Export Exception", e)
        raise
    else:
        return gen_export.id


@task()
def create_zip_export(username, id_string, export_id, query=None):
    export = Export.objects.get(id=export_id)
    try:
        gen_export = generate_attachments_zip_export(
            Export.ZIP_EXPORT, 'zip', username, id_string, export_id, query)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        report_exception("ZIP Export Exception", e)
        raise
    else:
        delete_export.apply_async(
            (), {'export_id': gen_export.id},
            countdown=settings.ZIP_EXPORT_COUNTDOWN)
        return gen_export.id


@task()
def create_csv_zip_export(username, id_string, export_id, query=None,
                          group_delimiter='/', split_select_multiples=True):
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(
            Export.CSV_ZIP_EXPORT, 'zip', username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        report_exception("CSV ZIP Export Exception", e)
        raise
    else:
        return gen_export.id


@task()
def delete_export(export_id):<commit_msg>a little more info for export exceptions<commit_after>import sys
from celery import task
from django.db import transaction
from django.conf import settings
from django.core.mail import mail_admins
from odk_viewer.models import Export
from utils.export_tools import generate_export,\
    generate_attachments_zip_export, generate_kml_export
from utils.logger_tools import mongo_sync_status, report_exception
from pandas_mongo_bridge import NoRecordsFoundError


def create_async_export(xform, export_type, query, force_xlsx, options=None):
    username = xform.user.username
    id_string = xform.id_string

    @transaction.commit_on_success
    def _create_export(xform, export_type):
        return Export.objects.create(xform=xform, export_type=export_type)

    export = _create_export(xform, export_type)
    result = None
    arguments = {
        'username': username,
        'id_string': id_string,
        'export_id': export.id,
        'query': query,
    }
    if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT,
                       Export.CSV_EXPORT, Export.CSV_ZIP_EXPORT]:
        if options and "group_delimiter" in options:
            arguments["group_delimiter"] = options["group_delimiter"]
        if options and "split_select_multiples" in options:
            arguments["split_select_multiples"] =\
                options["split_select_multiples"]

        # start async export
        if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT]:
            result = create_xls_export.apply_async((), arguments, countdown=10)
        elif export_type == Export.CSV_EXPORT:
            result = create_csv_export.apply_async(
                (), arguments, countdown=10)
        elif export_type == Export.CSV_ZIP_EXPORT:
            result = create_csv_zip_export.apply_async(
                (), arguments, countdown=10)
        else:
            raise Export.ExportTypeError
    elif export_type == Export.ZIP_EXPORT:
        # start async export
        result = create_zip_export.apply_async(
            (), arguments, countdown=10)
    elif export_type == Export.KML_EXPORT:
        # start async export
        result = create_kml_export.apply_async(
            (), arguments, countdown=10)
    else:
        raise Export.ExportTypeError
    if result:
        # when celery is running eager, the export has been generated by the
        # time we get here so lets retrieve the export object a fresh before we
        # save
        if settings.CELERY_ALWAYS_EAGER:
            export = Export.objects.get(id=export.id)
        export.task_id = result.task_id
        export.save()
        return export, result
    return None


@task()
def create_xls_export(username, id_string, export_id, query=None,
                      force_xlsx=True, group_delimiter='/',
                      split_select_multiples=True):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    ext = 'xls' if not force_xlsx else 'xlsx'

    export = Export.objects.get(id=export_id)
    # though export is not available when for has 0 submissions, we
    # catch this since it potentially stops celery
    try:
        gen_export = generate_export(
            Export.XLS_EXPORT, ext, username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception("XLS Export Exception: Export ID - "
                         "%(export_id)s, /%(username)s/%(id_string)s"
                         % details, e, sys.exc_info())
        #raise for now to let celery know we failed
        # - doesnt seem to break celery`
        raise
    else:
        return gen_export.id


@task()
def create_csv_export(username, id_string, export_id, query=None,
                      group_delimiter='/', split_select_multiples=True):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(
            Export.CSV_EXPORT, 'csv', username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception("CSV Export Exception: Export ID - "
                         "%(export_id)s, /%(username)s/%(id_string)s"
                         % details, e, sys.exc_info())
        raise
    else:
        return gen_export.id


@task()
def create_kml_export(username, id_string, export_id, query=None):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state

    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_kml_export(
            Export.KML_EXPORT, 'kml', username, id_string, export_id, query)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception("KML Export Exception: Export ID - "
                         "%(export_id)s, /%(username)s/%(id_string)s"
                         % details, e, sys.exc_info())
        raise
    else:
        return gen_export.id


@task()
def create_zip_export(username, id_string, export_id, query=None):
    export = Export.objects.get(id=export_id)
    try:
        gen_export = generate_attachments_zip_export(
            Export.ZIP_EXPORT, 'zip', username, id_string, export_id, query)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception("Zip Export Exception: Export ID - "
                         "%(export_id)s, /%(username)s/%(id_string)s"
                         % details, e)
        raise
    else:
        delete_export.apply_async(
            (), {'export_id': gen_export.id},
            countdown=settings.ZIP_EXPORT_COUNTDOWN)
        return gen_export.id


@task()
def create_csv_zip_export(username, id_string, export_id, query=None,
                          group_delimiter='/', split_select_multiples=True):
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(
            Export.CSV_ZIP_EXPORT, 'zip', username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception("CSV ZIP Export Exception: Export ID - "
                         "%(export_id)s, /%(username)s/%(id_string)s"
                         % details, e, sys.exc_info())
        raise
    else:
        return gen_export.id


@task()
def delete_export(export_id):
{'fraction_non_alphanumeric': 0.08290535583272193, 'fraction_numerical': 0.0018675381844860935, 'mean_word_length': 2.5114754098360654, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4184201', 'n_tokens_mistral': 4420, 'n_tokens_neox': 4218, 'n_words': 1091}