<commit_before>"""
Copyright [2009-2015] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import datetime
import django_rq
import gzip
import json
import os
import re
import requests
import subprocess as sub
import tempfile

from django.conf import settings
from django.core.servers.basehttp import FileWrapper
from django.http import JsonResponse, StreamingHttpResponse
from django.utils.text import get_valid_filename
from django.views.decorators.cache import never_cache

from contextlib import closing
from rq import get_current_job
from rest_framework import renderers

from apiv1.serializers import RnaFlatSerializer
from portal.models import Rna
from settings import EBI_SEARCH_ENDPOINT, EXPIRATION, MAX_RUN_TIME, ESLSFETCH, \
                     FASTA_DB


def export_search_results(query, _format, hits):
    """
    RQ worker function.

    * paginate over EBI search results results
    * extract RNAcentral ids
    * write the data to a local file in the specified format
    * return the filename
    """
    def get_results_page(start, end):
        """
        Retrieve a page of search results and return RNAcentral ids.
        """
        rnacentral_ids = []
        page_size = end - start
        url = ''.join([EBI_SEARCH_ENDPOINT,
                      '?query={query}',
                      '&start={start}',
                      '&size={page_size}',
                      '&format=json']).format(query=query, start=start,
                                              page_size=page_size)
        data = json.loads(requests.get(url).text)
        for entry in data['entries']:
            rnacentral_ids.append(re.sub(r'_\d+$', '', entry['id']))
        return rnacentral_ids

    def format_output(rnacentral_ids):
        """
        Given a list of RNAcentral ids, return the results
        in the specified format.
        """
        if _format == 'list':
            return '\n'.join(rnacentral_ids) + '\n'
        elif _format == 'json':
            queryset = Rna.objects.filter(upi__in=rnacentral_ids).all()
            serializer = RnaFlatSerializer(queryset, many=True)
            renderer = renderers.JSONRenderer()
            output = renderer.render(serializer.data)
            # omit opening and closing square brackets for easy concatenation
            output = output[1:-1]
            # make relative urls absolute
            output = output.replace('"/api/v1/', '"http://rnacentral.org/api/v1/')
        return output

    def run_easel(temp_file, filename):
        """
        Export RNAcentral ids saved in a temporary file using Easel esl-sfetch
        for accessing the FASTA database.
        """
        # make sure that temporary file is saved to disk
        temp_file.flush()
        os.fsync(temp_file.fileno())
        cmd = '{esl_binary} -f {fasta_db} {id_list} | gzip > {output}'.format(
            esl_binary=ESLSFETCH,
            fasta_db=FASTA_DB,
            id_list=temp_file.name,
            output=filename)
        process = sub.Popen(cmd, stdout=sub.PIPE, stderr=sub.PIPE, shell=True)
        output, errors = process.communicate()
        return_code = process.returncode
        temp_file.close()
        if return_code != 0:
            class EaselError(Exception):
                """Raise when Easel exits with a non-zero status"""
                pass
            raise EaselError(errors, output + '\n' + cmd, return_code)

    def paginate_over_results():
        """
        Paginate over the results and write out the data to an archive.
        JSON requires special treatment in order to concatenate
        multiple batches
        """
        filename = os.path.join(settings.EXPORT_RESULTS_DIR,
                                '%s.%s.gz' % (job.id, _format))
        start = 0
        page_size = 100 # max EBI search page size

        if _format in ['json', 'list']:
            archive = gzip.open(filename, 'wb')
        if _format == 'json':
            archive.write('[')
        if _format == 'fasta':
            f = tempfile.NamedTemporaryFile(delete=True,
                dir=settings.EXPORT_RESULTS_DIR)

        while start < hits:
            max_end = start + page_size
            end = min(max_end, hits)
            rnacentral_ids = get_results_page(start, end)
            if _format == 'fasta':
                # write out RNAcentral ids to a temporary file
                for _id in rnacentral_ids:
                    f.write('{0}\n'.format(_id))
            if _format in ['json', 'list']:
                text = format_output(rnacentral_ids)
                archive.write(text)
            if _format == 'json' and end != hits:
                # join batches with commas except for the last iteration
                archive.write(',\n')
            job.meta['progress'] = round(float(start) * 100 / hits, 2)
            start = end
            job.save()

        if _format == 'json':
            archive.write(']')
            archive.close()
        if _format == 'fasta':
            run_easel(f, filename)
        job.meta['progress'] = 100
        job.save()
        return filename

    job = get_current_job()
    filename = paginate_over_results()
    return filename


def get_job(job_id):
    """
    Get job from local or remote queues.

    Return a tuple (job, remote_server), where
    * `job` - job object
    * `remote_server` - server where the job was run
                        (None for localhost)
    """
    rq_queues = getattr(settings, 'RQ_QUEUES', [])
    for queue_id, params in rq_queues.iteritems():
        queue = django_rq.get_queue(queue_id)
        job = queue.fetch_job(job_id)
        if job:
            return (job, params['REMOTE_SERVER'])<commit_msg>Cap progress completion until easel export is done<commit_after>"""
Copyright [2009-2015] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import datetime
import django_rq
import gzip
import json
import os
import re
import requests
import subprocess as sub
import tempfile

from django.conf import settings
from django.core.servers.basehttp import FileWrapper
from django.http import JsonResponse, StreamingHttpResponse
from django.utils.text import get_valid_filename
from django.views.decorators.cache import never_cache

from contextlib import closing
from rq import get_current_job
from rest_framework import renderers

from apiv1.serializers import RnaFlatSerializer
from portal.models import Rna
from settings import EBI_SEARCH_ENDPOINT, EXPIRATION, MAX_RUN_TIME, ESLSFETCH, \
                     FASTA_DB


def export_search_results(query, _format, hits):
    """
    RQ worker function.

    * paginate over EBI search results results
    * extract RNAcentral ids
    * write the data to a local file in the specified format
    * return the filename
    """
    def get_results_page(start, end):
        """
        Retrieve a page of search results and return RNAcentral ids.
        """
        rnacentral_ids = []
        page_size = end - start
        url = ''.join([EBI_SEARCH_ENDPOINT,
                      '?query={query}',
                      '&start={start}',
                      '&size={page_size}',
                      '&format=json']).format(query=query, start=start,
                                              page_size=page_size)
        data = json.loads(requests.get(url).text)
        for entry in data['entries']:
            rnacentral_ids.append(re.sub(r'_\d+$', '', entry['id']))
        return rnacentral_ids

    def format_output(rnacentral_ids):
        """
        Given a list of RNAcentral ids, return the results
        in the specified format.
        """
        if _format == 'list':
            return '\n'.join(rnacentral_ids) + '\n'
        elif _format == 'json':
            queryset = Rna.objects.filter(upi__in=rnacentral_ids).all()
            serializer = RnaFlatSerializer(queryset, many=True)
            renderer = renderers.JSONRenderer()
            output = renderer.render(serializer.data)
            # omit opening and closing square brackets for easy concatenation
            output = output[1:-1]
            # make relative urls absolute
            output = output.replace('"/api/v1/', '"http://rnacentral.org/api/v1/')
        return output

    def run_easel(temp_file, filename):
        """
        Export RNAcentral ids saved in a temporary file using Easel esl-sfetch
        for accessing the FASTA database.
        """
        # make sure that temporary file is saved to disk
        temp_file.flush()
        os.fsync(temp_file.fileno())
        cmd = '{esl_binary} -f {fasta_db} {id_list} | gzip > {output}'.format(
            esl_binary=ESLSFETCH,
            fasta_db=FASTA_DB,
            id_list=temp_file.name,
            output=filename)
        process = sub.Popen(cmd, stdout=sub.PIPE, stderr=sub.PIPE, shell=True)
        output, errors = process.communicate()
        return_code = process.returncode
        temp_file.close()
        if return_code != 0:
            class EaselError(Exception):
                """Raise when Easel exits with a non-zero status"""
                pass
            raise EaselError(errors, output + '\n' + cmd, return_code)

    def paginate_over_results():
        """
        Paginate over the results and write out the data to an archive.
        JSON requires special treatment in order to concatenate
        multiple batches
        """
        filename = os.path.join(settings.EXPORT_RESULTS_DIR,
                                '%s.%s.gz' % (job.id, _format))
        start = 0
        page_size = 100 # max EBI search page size

        if _format in ['json', 'list']:
            archive = gzip.open(filename, 'wb')
        if _format == 'json':
            archive.write('[')
        if _format == 'fasta':
            f = tempfile.NamedTemporaryFile(delete=True,
                dir=settings.EXPORT_RESULTS_DIR)

        while start < hits:
            max_end = start + page_size
            end = min(max_end, hits)
            rnacentral_ids = get_results_page(start, end)
            if _format == 'fasta':
                # write out RNAcentral ids to a temporary file
                for _id in rnacentral_ids:
                    f.write('{0}\n'.format(_id))
            if _format in ['json', 'list']:
                text = format_output(rnacentral_ids)
                archive.write(text)
            if _format == 'json' and end != hits:
                # join batches with commas except for the last iteration
                archive.write(',\n')
            start = end
            job.meta['progress'] = min(round(float(start) * 100 / hits, 2), 85)
            job.save()

        if _format == 'json':
            archive.write(']')
            archive.close()
        if _format == 'fasta':
            run_easel(f, filename)
        job.meta['progress'] = 100
        job.save()
        return filename

    job = get_current_job()
    filename = paginate_over_results()
    return filename


def get_job(job_id):
    """
    Get job from local or remote queues.

    Return a tuple (job, remote_server), where
    * `job` - job object
    * `remote_server` - server where the job was run
                        (None for localhost)
    """
    rq_queues = getattr(settings, 'RQ_QUEUES', [])
    for queue_id, params in rq_queues.iteritems():
        queue = django_rq.get_queue(queue_id)
        job = queue.fetch_job(job_id)
        if job:
            return (job, params['REMOTE_SERVER'])
{'fraction_non_alphanumeric': 0.0916, 'fraction_numerical': 0.00496, 'mean_word_length': 2.160809102402023, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2236555', 'n_tokens_mistral': 3603, 'n_tokens_neox': 3418, 'n_words': 1057}