<commit_before>import datetime
import json
from urlparse import urlsplit

from django.contrib.auth.models import User
from django.db import connection
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.timezone import utc
from rest_framework import status, viewsets
from rest_framework.decorators import detail_route, list_route
from rest_framework.permissions import IsAuthenticated, AllowAny
from rest_framework.response import Response
from rest_framework.views import APIView
from ws4redis.publisher import RedisPublisher
from ws4redis.redis_store import RedisMessage

from crowdsourcing import constants
from crowdsourcing.models import Task, TaskWorker, TaskWorkerResult, UserPreferences, ReturnFeedback
from crowdsourcing.permissions.task import HasExceededReservedLimit, IsTaskOwner
from crowdsourcing.permissions.util import IsSandbox
from crowdsourcing.serializers.project import ProjectSerializer
from crowdsourcing.serializers.task import *
from crowdsourcing.tasks import update_worker_cache, post_approve, refund_task
from crowdsourcing.utils import get_model_or_none
from mturk.tasks import mturk_hit_update, mturk_approve


class TaskViewSet(viewsets.ModelViewSet):
    queryset = Task.objects.all()
    serializer_class = TaskSerializer
    filter_params = ['project_id', 'rerun_key', 'batch_id']

    def list(self, request, *args, **kwargs):
        try:
            by = request.query_params.get('filter_by', 'project_id')
            if by not in self.filter_params:
                return Response(data={"message": "Invalid filter by field."}, status=status.HTTP_400_BAD_REQUEST)
            filter_value = request.query_params.get(by, -1)
            task = Task.objects.filter(**{by: filter_value}).order_by('row_number')
            task_serialized = TaskSerializer(task, many=True,
                                             fields=('id', 'data', 'batch', 'project_data', 'row_number'))
            return Response(task_serialized.data)
        except:
            return Response([])

    @list_route(methods=['get'], url_path='list-data')
    def list_conflicts(self, request, *args, **kwargs):
        project = request.query_params.get('project', -1)
        offset = int(request.query_params.get('offset', 0))

        # noinspection SqlResolve
        query = '''
            SELECT t.id, t.data, t.row_number
            FROM crowdsourcing_task t
              INNER JOIN (

                           SELECT t.group_id, count(tw.id)
                           FROM crowdsourcing_task t
                             LEFT OUTER JOIN crowdsourcing_taskworker tw ON (t.id = tw.task_id
                             AND tw.status NOT IN (4, 6, 7))
                           WHERE exclude_at IS NULL AND t.deleted_at IS NULL AND t.project_id <> (%(project_id)s)
                           GROUP BY t.group_id HAVING count(tw.id)>0) all_tasks ON all_tasks.group_id = t.group_id
            WHERE project_id = (%(project_id)s) AND deleted_at IS NULL
            LIMIT 10 OFFSET (%(seek)s)
        '''

        tasks = list(Task.objects.raw(query, params={'project_id': project, 'seek': offset}))
        headers = []
        if len(tasks) > 0:
            headers = tasks[0].data.keys()[:4]
        serializer = TaskSerializer(tasks, many=True, fields=('id', 'data', 'row_number'))
        return Response({'headers': headers, 'tasks': serializer.data})

    def retrieve(self, request, *args, **kwargs):
        object = self.get_object()
        serializer = TaskSerializer(instance=object, fields=('id', 'template', 'project_data',
                                                             'worker_count', 'completed', 'total'))
        return Response(data=serializer.data, status=status.HTTP_200_OK)

    def destroy(self, request, *args, **kwargs):
        task = self.get_object()
        task.delete()
        return Response({'status': 'deleted task'})

    @detail_route(methods=['get'])
    def retrieve_with_data(self, request, *args, **kwargs):
        task = self.get_object()
        task_worker = TaskWorker.objects.filter(worker=request.user, task=task).first()
        serializer = TaskSerializer(instance=task,
                                    fields=('id', 'template', 'project_data', 'status', 'has_comments'),
                                    context={'task_worker': task_worker})
        requester_alias = task.project.owner.username
        project = task.project.id
        target = task.project.owner.id
        timeout = task.project.timeout
        worker_timestamp = task_worker.created_at
        now = datetime.datetime.utcnow().replace(tzinfo=utc)
        if timeout is not None:
            time_left = int((timeout * 60) - (now - worker_timestamp).total_seconds())
        else:
            time_left = None

        auto_accept = False
        user_prefs = get_model_or_none(UserPreferences, user=request.user)
        if user_prefs is not None:
            auto_accept = user_prefs.auto_accept

        return Response({'data': serializer.data,
                         'requester_alias': requester_alias,
                         'project': project,
                         'time_left': time_left,
                         'auto_accept': auto_accept,
                         'task_worker_id': task_worker.id,
                         'target': target}, status.HTTP_200_OK)

    @list_route(methods=['get'])
    def list_by_project(self, request, **kwargs):
        tasks = Task.objects.filter(project=request.query_params.get('project_id')).order_by('id')
        task_serializer = TaskSerializer(instance=tasks, many=True, fields=('id', 'updated_at',
                                                                            'worker_count', 'completed', 'total'))
        return Response(data=task_serializer.data, status=status.HTTP_200_OK)

    @detail_route(methods=['get'])
    def list_comments(self, request, **kwargs):
        comments = models.TaskComment.objects.filter(task=kwargs['pk'])
        serializer = TaskCommentSerializer(instance=comments, many=True, fields=('comment', 'id',))
        response_data = {
            'task': kwargs['pk'],
            'comments': serializer.data
        }
        return Response(response_data, status.HTTP_200_OK)

    @detail_route(methods=['post'])
    def post_comment(self, request, **kwargs):
        serializer = TaskCommentSerializer(data=request.data)
        task_comment_data = {}
        if serializer.is_valid():
            comment = serializer.create(task=kwargs['pk'], sender=request.user)
            task_comment_data = TaskCommentSerializer(comment, fields=('id', 'comment',)).data

        return Response(task_comment_data, status.HTTP_200_OK)

    @list_route(methods=['post'], url_path='relaunch-all')
    def relaunch_all(self, request, *args, **kwargs):
        project_id = request.query_params.get('project', -1)
        project = get_object_or_404(models.Project, pk=project_id)
        tasks = models.Task.objects.active().filter(~Q(project_id=project_id), project__group_id=project.group_id)
        self.serializer_class().bulk_update(tasks, {'exclude_at': project_id})
        return Response(data={}, status=status.HTTP_200_OK)

    @detail_route(methods=['post'], url_path='relaunch')
    def relaunch(self, request, *args, **kwargs):
        task = self.get_object()
        tasks = models.Task.objects.active().filter(~Q(id=task.id), group_id=task.group_id)
        self.serializer_class().bulk_update(tasks, {'exclude_at': task.project_id})
        return Response(data={}, status=status.HTTP_200_OK)

    @detail_route(methods=['get'], url_path='is-done')
    def is_done(self, request, *args, **kwargs):
        group_id = self.get_object().group_id
        # noinspection SqlResolve
        query = '''
            SELECT count(t.id) remaining
            FROM crowdsourcing_task t
              INNER JOIN (SELECT
                            group_id,
                            max(id) id
                          FROM crowdsourcing_task
                          WHERE deleted_at IS NULL
                          GROUP BY group_id) t_max ON t_max.id = t.id
              INNER JOIN crowdsourcing_project p ON p.id = t.project_id
              INNER JOIN (
                           SELECT
                             t.group_id,
                             sum(t.others) OTHERS
                           FROM (
                                  SELECT
                                    t.group_id,
                                    CASE WHEN tw.id IS NOT NULL
                                      THEN 1
                                    ELSE 0 END OTHERS
                                  FROM crowdsourcing_task t
                                    LEFT OUTER JOIN crowdsourcing_taskworker tw
                                      ON (t.id = tw.task_id AND tw.status IN (2, 3))
                                  WHERE t.exclude_at IS NULL AND t.deleted_at IS NULL) t
                           GROUP BY t.group_id) t_count ON t_count.group_id = t.group_id
            WHERE t_count.others < p.repetition AND t.group_id = (%(group_id)s);
        '''
        cursor = connection.cursor()
        cursor.execute(query, {'group_id': group_id})
        remaining_count = cursor.fetchall()[0][0] if cursor.rowcount > 0 else 0
        return Response(data={"is_done": remaining_count == 0}, status=status.HTTP_200_OK)


class TaskWorkerViewSet(viewsets.ModelViewSet):<commit_msg>add expected number of submissions to is_done call<commit_after>import datetime
import json
from urlparse import urlsplit

from django.contrib.auth.models import User
from django.db import connection
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.timezone import utc
from rest_framework import status, viewsets
from rest_framework.decorators import detail_route, list_route
from rest_framework.permissions import IsAuthenticated, AllowAny
from rest_framework.response import Response
from rest_framework.views import APIView
from ws4redis.publisher import RedisPublisher
from ws4redis.redis_store import RedisMessage

from crowdsourcing import constants
from crowdsourcing.models import Task, TaskWorker, TaskWorkerResult, UserPreferences, ReturnFeedback
from crowdsourcing.permissions.task import HasExceededReservedLimit, IsTaskOwner
from crowdsourcing.permissions.util import IsSandbox
from crowdsourcing.serializers.project import ProjectSerializer
from crowdsourcing.serializers.task import *
from crowdsourcing.tasks import update_worker_cache, post_approve, refund_task
from crowdsourcing.utils import get_model_or_none
from mturk.tasks import mturk_hit_update, mturk_approve


class TaskViewSet(viewsets.ModelViewSet):
    queryset = Task.objects.all()
    serializer_class = TaskSerializer
    filter_params = ['project_id', 'rerun_key', 'batch_id']

    def list(self, request, *args, **kwargs):
        try:
            by = request.query_params.get('filter_by', 'project_id')
            if by not in self.filter_params:
                return Response(data={"message": "Invalid filter by field."}, status=status.HTTP_400_BAD_REQUEST)
            filter_value = request.query_params.get(by, -1)
            task = Task.objects.filter(**{by: filter_value}).order_by('row_number')
            task_serialized = TaskSerializer(task, many=True,
                                             fields=('id', 'data', 'batch', 'project_data', 'row_number'))
            return Response(task_serialized.data)
        except:
            return Response([])

    @list_route(methods=['get'], url_path='list-data')
    def list_conflicts(self, request, *args, **kwargs):
        project = request.query_params.get('project', -1)
        offset = int(request.query_params.get('offset', 0))

        # noinspection SqlResolve
        query = '''
            SELECT t.id, t.data, t.row_number
            FROM crowdsourcing_task t
              INNER JOIN (

                           SELECT t.group_id, count(tw.id)
                           FROM crowdsourcing_task t
                             LEFT OUTER JOIN crowdsourcing_taskworker tw ON (t.id = tw.task_id
                             AND tw.status NOT IN (4, 6, 7))
                           WHERE exclude_at IS NULL AND t.deleted_at IS NULL AND t.project_id <> (%(project_id)s)
                           GROUP BY t.group_id HAVING count(tw.id)>0) all_tasks ON all_tasks.group_id = t.group_id
            WHERE project_id = (%(project_id)s) AND deleted_at IS NULL
            LIMIT 10 OFFSET (%(seek)s)
        '''

        tasks = list(Task.objects.raw(query, params={'project_id': project, 'seek': offset}))
        headers = []
        if len(tasks) > 0:
            headers = tasks[0].data.keys()[:4]
        serializer = TaskSerializer(tasks, many=True, fields=('id', 'data', 'row_number'))
        return Response({'headers': headers, 'tasks': serializer.data})

    def retrieve(self, request, *args, **kwargs):
        object = self.get_object()
        serializer = TaskSerializer(instance=object, fields=('id', 'template', 'project_data',
                                                             'worker_count', 'completed', 'total'))
        return Response(data=serializer.data, status=status.HTTP_200_OK)

    def destroy(self, request, *args, **kwargs):
        task = self.get_object()
        task.delete()
        return Response({'status': 'deleted task'})

    @detail_route(methods=['get'])
    def retrieve_with_data(self, request, *args, **kwargs):
        task = self.get_object()
        task_worker = TaskWorker.objects.filter(worker=request.user, task=task).first()
        serializer = TaskSerializer(instance=task,
                                    fields=('id', 'template', 'project_data', 'status', 'has_comments'),
                                    context={'task_worker': task_worker})
        requester_alias = task.project.owner.username
        project = task.project.id
        target = task.project.owner.id
        timeout = task.project.timeout
        worker_timestamp = task_worker.created_at
        now = datetime.datetime.utcnow().replace(tzinfo=utc)
        if timeout is not None:
            time_left = int((timeout * 60) - (now - worker_timestamp).total_seconds())
        else:
            time_left = None

        auto_accept = False
        user_prefs = get_model_or_none(UserPreferences, user=request.user)
        if user_prefs is not None:
            auto_accept = user_prefs.auto_accept

        return Response({'data': serializer.data,
                         'requester_alias': requester_alias,
                         'project': project,
                         'time_left': time_left,
                         'auto_accept': auto_accept,
                         'task_worker_id': task_worker.id,
                         'target': target}, status.HTTP_200_OK)

    @list_route(methods=['get'])
    def list_by_project(self, request, **kwargs):
        tasks = Task.objects.filter(project=request.query_params.get('project_id')).order_by('id')
        task_serializer = TaskSerializer(instance=tasks, many=True, fields=('id', 'updated_at',
                                                                            'worker_count', 'completed', 'total'))
        return Response(data=task_serializer.data, status=status.HTTP_200_OK)

    @detail_route(methods=['get'])
    def list_comments(self, request, **kwargs):
        comments = models.TaskComment.objects.filter(task=kwargs['pk'])
        serializer = TaskCommentSerializer(instance=comments, many=True, fields=('comment', 'id',))
        response_data = {
            'task': kwargs['pk'],
            'comments': serializer.data
        }
        return Response(response_data, status.HTTP_200_OK)

    @detail_route(methods=['post'])
    def post_comment(self, request, **kwargs):
        serializer = TaskCommentSerializer(data=request.data)
        task_comment_data = {}
        if serializer.is_valid():
            comment = serializer.create(task=kwargs['pk'], sender=request.user)
            task_comment_data = TaskCommentSerializer(comment, fields=('id', 'comment',)).data

        return Response(task_comment_data, status.HTTP_200_OK)

    @list_route(methods=['post'], url_path='relaunch-all')
    def relaunch_all(self, request, *args, **kwargs):
        project_id = request.query_params.get('project', -1)
        project = get_object_or_404(models.Project, pk=project_id)
        tasks = models.Task.objects.active().filter(~Q(project_id=project_id), project__group_id=project.group_id)
        self.serializer_class().bulk_update(tasks, {'exclude_at': project_id})
        return Response(data={}, status=status.HTTP_200_OK)

    @detail_route(methods=['post'], url_path='relaunch')
    def relaunch(self, request, *args, **kwargs):
        task = self.get_object()
        tasks = models.Task.objects.active().filter(~Q(id=task.id), group_id=task.group_id)
        self.serializer_class().bulk_update(tasks, {'exclude_at': task.project_id})
        return Response(data={}, status=status.HTTP_200_OK)

    @detail_route(methods=['get'], url_path='is-done')
    def is_done(self, request, *args, **kwargs):
        group_id = self.get_object().group_id
        # noinspection SqlResolve
        query = '''
            SELECT greatest(t_count.completed, p.repetition) expected, t_count.completed, p.repetition
            FROM crowdsourcing_task t
              INNER JOIN (SELECT
                            group_id,
                            max(id) id
                          FROM crowdsourcing_task
                          WHERE deleted_at IS NULL
                          GROUP BY group_id) t_max ON t_max.id = t.id
              INNER JOIN crowdsourcing_project p ON p.id = t.project_id
              INNER JOIN (
                           SELECT
                             t.group_id,
                             coalesce(sum(t.others), 0) completed
                           FROM (
                                  SELECT
                                    t.group_id,
                                    CASE WHEN tw.id IS NOT NULL
                                      THEN 1
                                    ELSE 0 END OTHERS
                                  FROM crowdsourcing_task t
                                    LEFT OUTER JOIN crowdsourcing_taskworker tw
                                      ON (t.id = tw.task_id AND tw.status IN (2, 3))
                                  WHERE t.exclude_at IS NULL AND t.deleted_at IS NULL) t
                           GROUP BY t.group_id) t_count ON t_count.group_id = t.group_id
            WHERE t.group_id = (%(group_id)s);
        '''
        cursor = connection.cursor()
        cursor.execute(query, {'group_id': group_id})
        task = cursor.fetchall()[0] if cursor.rowcount > 0 else None
        done = task[1] >= task[2]
        return Response(data={"is_done": done, 'expected': task[0]},
                        status=status.HTTP_200_OK)


class TaskWorkerViewSet(viewsets.ModelViewSet):
{'fraction_non_alphanumeric': 0.1025654368527866, 'fraction_numerical': 0.006192433782588333, 'mean_word_length': 2.247380871916188, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5370776', 'n_tokens_mistral': 5398, 'n_tokens_neox': 5164, 'n_words': 1169}