<commit_before># -*- coding: utf-8 -*-

"""
download.py

.. moduleauthor::  <NAME> <<EMAIL>>
.. moduleauthor::  <NAME> <<EMAIL>>

"""
import logging
import shutil
import subprocess
import urllib
from itertools import chain
from tempfile import mkstemp, NamedTemporaryFile
from threading import Thread
from urllib2 import urlopen, URLError

from os import path, devnull, unlink

from django.core.servers.basehttp import FileWrapper
from django.http import HttpResponse, HttpResponseRedirect, \
    HttpResponseNotFound
from django.conf import settings

from tardis.tardis_portal.models import *
from tardis.tardis_portal.auth.decorators import *
from tardis.tardis_portal.views import return_response_not_found, \
    return_response_error

logger = logging.getLogger(__name__)

class StreamingFile:

    def __init__(self, writer_callable):
        self.runnable = writer_callable
        _, self.name = mkstemp()
        self.thread = Thread(target=self)
        self.thread.start()
        self.reader = open(self.name, 'rb')

    def __call__(self):
        self.runnable(self.name)

    def read(self, size=1):
        def read_file(size):
            return self.reader.read(size)

        remaining_bytes = size
        buf = ''
        while self.thread.is_alive() and remaining_bytes > 0:
            from time import sleep
            sleep(0.01)
            read_bytes = read_file(remaining_bytes)
            buf += read_bytes
            remaining_bytes -= len(read_bytes)

        if remaining_bytes > 0:
            buf += read_file(remaining_bytes)

        return buf

    def close(self):
        self.reader.close()
        unlink(self.name)

def _create_download_response(request, datafile_id, disposition='attachment'):
    # Get datafile (and return 404 if absent)
    try:
        datafile = Dataset_File.objects.get(pk=datafile_id)
    except Dataset_File.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        dataset_file_id=datafile.id):
        return return_response_error(request)
    # Send an image that can be seen in the browser
    if disposition == 'inline' and datafile.is_image():
        from tardis.tardis_portal.iiif import download_image
        args = (request, datafile.id, 'full', 'full', '0', 'native')
        # Send unconverted image if web-compatible
        if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'):
            return download_image(*args)
        # Send converted image
        return download_image(*args, format='png')
    # Send local file
    try:
        # Get file object for datafile
        file_obj = datafile.get_file()
        if not file_obj:
            # If file path doesn't resolve, return not found
            return return_response_not_found(request)
        wrapper = FileWrapper(file_obj)
        response = HttpResponse(wrapper,
                                mimetype=datafile.get_mimetype())
        response['Content-Disposition'] = \
            '%s; filename="%s"' % (disposition, datafile.filename)
        return response
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)

def view_datafile(request, datafile_id):
    return _create_download_response(request, datafile_id, 'inline')

def download_datafile(request, datafile_id):
    return _create_download_response(request, datafile_id)

def _get_filename(rootdir, df):
    return path.join(rootdir, str(df.dataset.id), df.filename)

def _get_datafile_details_for_archive(rootdir, datafiles):
    return [(df.get_file(), _get_filename(rootdir, df)) \
             for df in datafiles]

def _write_files_to_archive(write_func, files):
    for fileObj, name in files:
        if not fileObj:
            logger.debug('Skipping %s - no file available.' % name)
            continue
        with NamedTemporaryFile(prefix='mytardis_tmp_dl_') as fdst:
            try:
                # Copy url to destination file
                shutil.copyfileobj(fileObj, fdst)
                # Flush the file so we can read from it properly
                fdst.flush()
                # Write file
                write_func(fdst.name, name)
            except URLError:
                logger.warn("Unable to fetch %s for archive download." % name)

def _write_tar_func(rootdir, datafiles):
    logger.debug('Getting files to write to archive')
    # Resolve url and name for the files
    files = _get_datafile_details_for_archive(rootdir, datafiles)
    logger.debug('Have %d files to write to archive' % len(files))
    # Define the function
    def write_tar(filename):
        from tarfile import TarFile
        tf = TarFile(filename, 'w')
        logger.debug('Writing tar archive to %s' % filename)
        _write_files_to_archive(tf.add, files)
        tf.close()
    # Returns the function to do the actual writing
    return write_tar

def _write_zip_func(rootdir, datafiles):
    logger.debug('Getting files to write to archive')
    # Resolve url and name for the files
    files = _get_datafile_details_for_archive(rootdir, datafiles)
    logger.debug('Have %d files to write to archive' % len(files))
    # Define the function
    def write_zip(filename):
        from zipfile import ZipFile
        zf = ZipFile(filename, 'w', allowZip64=True)
        logger.debug('Writing zip archive to %s' % filename)
        _write_files_to_archive(zf.write, files)
        zf.close()
    # Returns the function to do the actual writing
    return write_zip


@experiment_download_required
def download_experiment(request, experiment_id, comptype):
    """
    takes string parameter "comptype" for compression method.
    Currently implemented: "zip" and "tar"
    """<commit_msg>Fix for bug that leaves a "permanently" full /tmp filesystem if download archive is too big.<commit_after># -*- coding: utf-8 -*-

"""
download.py

.. moduleauthor::  <NAME> <<EMAIL>>
.. moduleauthor::  <NAME> <<EMAIL>>

"""
import logging
import shutil
import subprocess
import urllib
from itertools import chain
from tempfile import mkstemp, NamedTemporaryFile
from threading import Thread
from urllib2 import urlopen, URLError

from os import path, devnull, unlink

from django.core.servers.basehttp import FileWrapper
from django.http import HttpResponse, HttpResponseRedirect, \
    HttpResponseNotFound
from django.conf import settings

from tardis.tardis_portal.models import *
from tardis.tardis_portal.auth.decorators import *
from tardis.tardis_portal.views import return_response_not_found, \
    return_response_error

logger = logging.getLogger(__name__)

class StreamingFile:

    def __init__(self, writer_callable):
        self.runnable = writer_callable
        _, self.name = mkstemp()
        self.thread = Thread(target=self)
        self.thread.start()
        self.reader = open(self.name, 'rb')

    def __call__(self):
        self.runnable(self.name)

    def read(self, size=1):
        def read_file(size):
            return self.reader.read(size)

        remaining_bytes = size
        buf = ''
        while self.thread.is_alive() and remaining_bytes > 0:
            from time import sleep
            sleep(0.01)
            read_bytes = read_file(remaining_bytes)
            buf += read_bytes
            remaining_bytes -= len(read_bytes)

        if remaining_bytes > 0:
            buf += read_file(remaining_bytes)

        return buf

    def close(self):
        self.reader.close()
        unlink(self.name)

def _create_download_response(request, datafile_id, disposition='attachment'):
    # Get datafile (and return 404 if absent)
    try:
        datafile = Dataset_File.objects.get(pk=datafile_id)
    except Dataset_File.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        dataset_file_id=datafile.id):
        return return_response_error(request)
    # Send an image that can be seen in the browser
    if disposition == 'inline' and datafile.is_image():
        from tardis.tardis_portal.iiif import download_image
        args = (request, datafile.id, 'full', 'full', '0', 'native')
        # Send unconverted image if web-compatible
        if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'):
            return download_image(*args)
        # Send converted image
        return download_image(*args, format='png')
    # Send local file
    try:
        # Get file object for datafile
        file_obj = datafile.get_file()
        if not file_obj:
            # If file path doesn't resolve, return not found
            return return_response_not_found(request)
        wrapper = FileWrapper(file_obj)
        response = HttpResponse(wrapper,
                                mimetype=datafile.get_mimetype())
        response['Content-Disposition'] = \
            '%s; filename="%s"' % (disposition, datafile.filename)
        return response
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)

def view_datafile(request, datafile_id):
    return _create_download_response(request, datafile_id, 'inline')

def download_datafile(request, datafile_id):
    return _create_download_response(request, datafile_id)

def _get_filename(rootdir, df):
    return path.join(rootdir, str(df.dataset.id), df.filename)

def _get_datafile_details_for_archive(rootdir, datafiles):
    return [(df.get_file(), _get_filename(rootdir, df)) \
             for df in datafiles]

def _write_files_to_archive(write_func, files):
    for fileObj, name in files:
        if not fileObj:
            logger.debug('Skipping %s - no file available.' % name)
            continue
        with NamedTemporaryFile(prefix='mytardis_tmp_dl_') as fdst:
            try:
                # Copy url to destination file
                shutil.copyfileobj(fileObj, fdst)
                # Flush the file so we can read from it properly
                fdst.flush()
                # Write file
                write_func(fdst.name, name)
            except URLError:
                logger.warn("Unable to fetch %s for archive download." % name)

def _write_tar_func(rootdir, datafiles):
    logger.debug('Getting files to write to archive')
    # Resolve url and name for the files
    files = _get_datafile_details_for_archive(rootdir, datafiles)
    logger.debug('Have %d files to write to archive' % len(files))
    # Define the function
    def write_tar(filename):
        from tarfile import TarFile
        try:
            tf = TarFile(filename, 'w')
            logger.debug('Writing tar archive to %s' % filename)
            _write_files_to_archive(tf.add, files)
            tf.close()
        except IOError as ex:
            logger.warn("I/O error({0}) while writing tar archive: {1}".format(e.errno, e.strerror))
            unlink(filename)
        finally:
            tf.close()
    # Returns the function to do the actual writing
    return write_tar

def _write_zip_func(rootdir, datafiles):
    logger.debug('Getting files to write to archive')
    # Resolve url and name for the files
    files = _get_datafile_details_for_archive(rootdir, datafiles)
    logger.debug('Have %d files to write to archive' % len(files))
    # Define the function
    def write_zip(filename):
        from zipfile import ZipFile
        try:
            zf = ZipFile(filename, 'w', allowZip64=True)
            logger.debug('Writing zip archive to %s' % filename)
            _write_files_to_archive(zf.write, files)
            zf.close()
        except IOError as ex:
            logger.warn("I/O error({0}) while writing zip archive: {1}".format(e.errno, e.strerror))
            unlink(filename)
        finally:
            zf.close()
    # Returns the function to do the actual writing
    return write_zip


@experiment_download_required
def download_experiment(request, experiment_id, comptype):
    """
    takes string parameter "comptype" for compression method.
    Currently implemented: "zip" and "tar"
    """
{'fraction_non_alphanumeric': 0.0868784192047032, 'fraction_numerical': 0.002612884788111374, 'mean_word_length': 2.720534629404617, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5980213', 'n_tokens_mistral': 3569, 'n_tokens_neox': 3505, 'n_words': 983}