<commit_before>import StringIO
import zipfile
from collections import OrderedDict
from django.http import HttpResponse
from django.db.models import Q
import pandas as pd
from rest_framework import status, mixins, serializers
from rest_framework.decorators import list_route
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.viewsets import GenericViewSet

from crowdsourcing.models import BatchFile, TaskWorker, TaskWorkerResult, Project
from crowdsourcing.serializers.file import BatchFileSerializer


class FileViewSet(mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, GenericViewSet):
    queryset = BatchFile.objects.filter(deleted_at__isnull=True)
    serializer_class = BatchFileSerializer
    permission_classes = [IsAuthenticated]

    def create(self, request, *args, **kwargs):
        serializer = BatchFileSerializer(data=request.data)
        if serializer.is_valid():
            batch_file = serializer.create()
            serializer = BatchFileSerializer(instance=batch_file)
            return Response(data=serializer.data, status=status.HTTP_201_CREATED)
        else:
            raise serializers.ValidationError(detail=serializer.errors)

    @list_route(methods=['get'], url_path='download-results')
    def download_results(self, request, *args, **kwargs):
        project_id = request.query_params.get('project_id', -1)
        project = Project.objects.filter(id=project_id, owner=request.user).first()
        if project is None:
            return Response([], status=status.HTTP_404_NOT_FOUND)

        revisions = Project.objects.prefetch_related('template', 'template__items').filter(
            ~Q(status=Project.STATUS_DRAFT), group_id=project.group_id).order_by('-id')

        if len(revisions) == 1 and revisions[0].template.items.filter(type='file_upload').count() == 0:
            data, rows = self._fetch_results(revisions[0].id)
            # http_status = status.HTTP_200_OK if rows > 0 else status.HTTP_204_NO_CONTENT
            resp = HttpResponse(data)
            resp['Content-Disposition'] = 'attachment; filename={}.csv'.format(revisions[0].name.replace(' ', '_'))
            resp['Content-Type'] = 'text/csv'
            return resp
            # return Response(data, http_status)
        else:
            zip_file_buffer = StringIO.StringIO()
            zip_file = zipfile.ZipFile(zip_file_buffer, "w")
            r = len(revisions)
            for rev in revisions:
                data, rows = self._fetch_results(rev.id)

                if rows > 0:
                    # file_upload_items = rev.template.items.filter(type='file_upload')
                    file_results = TaskWorkerResult.objects \
                        .prefetch_related('attachment').filter(attachment__isnull=False,
                                                               task_worker__task__project_id=rev.id)
                    zip_file.writestr('{}/revision_{}({}).csv'.format(revisions[0].name.replace(' ', '_'), r, rev.id),
                                      data)
                    for f in file_results:
                        zip_file.writestr(
                            '{}/responses/{}_{}'.format(revisions[0].name.replace(' ', '_'), f.task_worker_id,
                                                        f.attachment.name),
                            f.attachment.file.read())
                r -= 1
            zip_file.close()
            resp = HttpResponse(zip_file_buffer.getvalue())
            resp['Content-Disposition'] = 'attachment; filename={}.zip'.format(revisions[0].name.replace(' ', '_'))<commit_msg>closes #922<commit_after>import StringIO
import zipfile
from collections import OrderedDict
from django.http import HttpResponse
from django.db.models import Q
import pandas as pd
from rest_framework import status, mixins, serializers
from rest_framework.decorators import list_route
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.viewsets import GenericViewSet

from crowdsourcing.models import BatchFile, TaskWorker, TaskWorkerResult, Project
from crowdsourcing.serializers.file import BatchFileSerializer


class FileViewSet(mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, GenericViewSet):
    queryset = BatchFile.objects.filter(deleted_at__isnull=True)
    serializer_class = BatchFileSerializer
    permission_classes = [IsAuthenticated]

    def create(self, request, *args, **kwargs):
        serializer = BatchFileSerializer(data=request.data)
        if serializer.is_valid():
            batch_file = serializer.create()
            serializer = BatchFileSerializer(instance=batch_file)
            return Response(data=serializer.data, status=status.HTTP_201_CREATED)
        else:
            raise serializers.ValidationError(detail=serializer.errors)

    @list_route(methods=['get'], url_path='download-results')
    def download_results(self, request, *args, **kwargs):
        project_id = request.query_params.get('project_id', -1)
        project = Project.objects.filter(id=project_id, owner=request.user).first()
        if project is None:
            return Response([], status=status.HTTP_404_NOT_FOUND)

        revisions = Project.objects.prefetch_related('template', 'template__items').filter(
            ~Q(status=Project.STATUS_DRAFT), group_id=project.group_id).order_by('-id')

        if len(revisions) == 1 and revisions[0].template.items.filter(type='file_upload').count() == 0:
            data, rows = self._fetch_results(revisions[0].id)
            # http_status = status.HTTP_200_OK if rows > 0 else status.HTTP_204_NO_CONTENT
            resp = HttpResponse(data)
            resp['Content-Disposition'] = 'attachment; filename={}.csv'.format(revisions[0].name.replace(' ', '_'))
            resp['Content-Type'] = 'text/csv'
            return resp
            # return Response(data, http_status)
        else:
            zip_file_buffer = StringIO.StringIO()
            zip_file = zipfile.ZipFile(zip_file_buffer, "w")
            r = len(revisions)
            for rev in revisions:
                data, rows = self._fetch_results(rev.id)

                if rows > 0:
                    # file_upload_items = rev.template.items.filter(type='file_upload')
                    file_results = TaskWorkerResult.objects \
                        .prefetch_related('attachment').filter(attachment__isnull=False,
                                                               task_worker__status__in=[TaskWorker.STATUS_SUBMITTED,
                                                                                        TaskWorker.STATUS_ACCEPTED,
                                                                                        TaskWorker.STATUS_RETURNED],
                                                               task_worker__task__project_id=rev.id)
                    zip_file.writestr('{}/revision_{}({}).csv'.format(revisions[0].name.replace(' ', '_'), r, rev.id),
                                      data)
                    for f in file_results:
                        zip_file.writestr(
                            '{}/responses/{}_{}'.format(revisions[0].name.replace(' ', '_'), f.task_worker_id,
                                                        f.attachment.name),
                            f.attachment.file.read())
                r -= 1
            zip_file.close()
            resp = HttpResponse(zip_file_buffer.getvalue())
            resp['Content-Disposition'] = 'attachment; filename={}.zip'.format(revisions[0].name.replace(' ', '_'))
{'fraction_non_alphanumeric': 0.10223144784639336, 'fraction_numerical': 0.006616502335236118, 'mean_word_length': 2.3014989293361885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2337421', 'n_tokens_mistral': 2063, 'n_tokens_neox': 1978, 'n_words': 366}