<commit_before>from celery.utils.log import get_task_logger
from unidecode import unidecode
from celery.task import task
import zipfile
from couchexport.models import Format, ExportSchema
import tempfile
import os
from soil.util import expose_download
from couchexport.export import SchemaMismatchException, ExportConfiguration

logging = get_task_logger(__name__)

@task
def export_async(custom_export, download_id, format=None, filename=None, **kwargs):
    try:
        tmp, checkpoint = custom_export.get_export_files(format=format, process=export_async, **kwargs)
    except SchemaMismatchException, e:
        # fire off a delayed force update to prevent this from happening again
        rebuild_schemas.delay(custom_export.index)
        msg = "Export failed for custom export {id}. Index is {index}. The specific error is {msg}."
        raise Exception(msg.format(id=custom_export._id,
                                   index=custom_export.index,
                                   msg=str(e)))

    try:
        format = tmp.format
    except AttributeError:
        pass
    if not filename:
        filename = custom_export.name
    return cache_file_to_be_served(tmp, checkpoint, download_id, format, filename)

@task
def rebuild_schemas(index):
    """
    Resets the schema for all checkpoints to the latest version based off the
    current document structure. Returns the number of checkpoints updated.
    """
    db = ExportSchema.get_db()
    all_checkpoints = ExportSchema.get_all_checkpoints(index)
    config = ExportConfiguration(db, index, disable_checkpoints=True)
    latest = config.create_new_checkpoint()
    for cp in all_checkpoints:
        cp.schema = latest.schema
        cp.save()
    return len(all_checkpoints)

@task
def bulk_export_async(bulk_export_helper, download_id,
                      filename="bulk_export", expiry=10*60*60, domain=None):


    if bulk_export_helper.zip_export:
        filename = "%s_%s"% (domain, filename) if domain else filename
        _, path = tempfile.mkstemp()
        os.close(_) 
        zf = zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED)
        try: 
            for file in bulk_export_helper.bulk_files:
                try:
                    bulk = file.generate_bulk_file()
                    zf.writestr("%s/%s" %(filename, file.filename), bulk.getvalue())
                except Exception as e:
                    logging.error("FAILED to add file to bulk export archive. %s" % e)
        finally:
            zf.close()

        return cache_file_to_be_served(
            tmp=path,
            checkpoint=bulk_export_helper,
            download_id=download_id,
            filename=filename,
            format='zip',
            expiry=expiry
        )
    else:
        export_object = bulk_export_helper.bulk_files[0]
        return cache_file_to_be_served(
            tmp=export_object.generate_bulk_file(),
            checkpoint=bulk_export_helper,
            download_id=download_id,
            filename=export_object.filename,
            format=export_object.format,
            expiry=expiry<commit_msg>delete bulk file zip when done with it<commit_after>from celery.utils.log import get_task_logger
from unidecode import unidecode
from celery.task import task
import zipfile
from couchexport.models import Format, ExportSchema
import tempfile
import os
from soil.util import expose_download
from couchexport.export import SchemaMismatchException, ExportConfiguration

logging = get_task_logger(__name__)

@task
def export_async(custom_export, download_id, format=None, filename=None, **kwargs):
    try:
        tmp, checkpoint = custom_export.get_export_files(format=format, process=export_async, **kwargs)
    except SchemaMismatchException, e:
        # fire off a delayed force update to prevent this from happening again
        rebuild_schemas.delay(custom_export.index)
        msg = "Export failed for custom export {id}. Index is {index}. The specific error is {msg}."
        raise Exception(msg.format(id=custom_export._id,
                                   index=custom_export.index,
                                   msg=str(e)))

    try:
        format = tmp.format
    except AttributeError:
        pass
    if not filename:
        filename = custom_export.name
    return cache_file_to_be_served(tmp, checkpoint, download_id, format, filename)

@task
def rebuild_schemas(index):
    """
    Resets the schema for all checkpoints to the latest version based off the
    current document structure. Returns the number of checkpoints updated.
    """
    db = ExportSchema.get_db()
    all_checkpoints = ExportSchema.get_all_checkpoints(index)
    config = ExportConfiguration(db, index, disable_checkpoints=True)
    latest = config.create_new_checkpoint()
    for cp in all_checkpoints:
        cp.schema = latest.schema
        cp.save()
    return len(all_checkpoints)

@task
def bulk_export_async(bulk_export_helper, download_id,
                      filename="bulk_export", expiry=10*60*60, domain=None):


    if bulk_export_helper.zip_export:
        filename = "%s_%s"% (domain, filename) if domain else filename
        _, path = tempfile.mkstemp()
        os.close(_) 
        zf = zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED)
        try: 
            for file in bulk_export_helper.bulk_files:
                try:
                    bulk = file.generate_bulk_file()
                    zf.writestr("%s/%s" %(filename, file.filename), bulk.getvalue())
                except Exception as e:
                    logging.error("FAILED to add file to bulk export archive. %s" % e)
        finally:
            zf.close()

        try:
            return cache_file_to_be_served(
                tmp=path,
                checkpoint=bulk_export_helper,
                download_id=download_id,
                filename=filename,
                format='zip',
                expiry=expiry
            )
        finally:
            os.remove(path)
    else:
        export_object = bulk_export_helper.bulk_files[0]
        return cache_file_to_be_served(
            tmp=export_object.generate_bulk_file(),
            checkpoint=bulk_export_helper,
            download_id=download_id,
            filename=export_object.filename,
            format=export_object.format,
            expiry=expiry
{'fraction_non_alphanumeric': 0.07677755005517893, 'fraction_numerical': 0.0022071574964527825, 'mean_word_length': 2.4972436604189636, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5962342', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1750, 'n_words': 437}