<commit_before>from crowdsourcing import models
from rest_framework import serializers
from crowdsourcing.serializers.dynamic import DynamicFieldsModelSerializer
from crowdsourcing.serializers.template import TemplateItemSerializer
from rest_framework.exceptions import ValidationError
from django.db import transaction
from crowdsourcing.serializers.template import TemplateSerializer
import json
from django.db.models import Count, F


class TaskWorkerResultListSerializer(serializers.ListSerializer):
    def create(self, **kwargs):
        for item in self.validated_data:
            models.TaskWorkerResult.objects.get_or_create(task_worker=kwargs['task_worker'], **item)


class TaskWorkerResultSerializer(DynamicFieldsModelSerializer):
    template_item_id = serializers.SerializerMethodField()

    class Meta:
        model = models.TaskWorkerResult
        list_serializer_class = TaskWorkerResultListSerializer
        fields = ('id', 'template_item', 'result', 'status', 'created_timestamp', 'last_updated', 'template_item_id')
        read_only_fields = ('created_timestamp', 'last_updated')

    def create(self, **kwargs):
        models.TaskWorkerResult.objects.get_or_create(self.validated_data)

    def get_template_item_id(self, obj):
        template_item = TemplateItemSerializer(instance=obj.template_item).data
        return template_item['id']


class TaskWorkerSerializer(DynamicFieldsModelSerializer):
    import multiprocessing
    
    lock = multiprocessing.Lock()
    task_worker_results = TaskWorkerResultSerializer(many=True, read_only=True)
    worker_alias = serializers.SerializerMethodField()
    task_worker_results_monitoring = serializers.SerializerMethodField()
    updated_delta = serializers.SerializerMethodField()
    requester_alias = serializers.SerializerMethodField()
    module = serializers.SerializerMethodField()
    project_name = serializers.SerializerMethodField()
    task_with_data_and_results = serializers.SerializerMethodField()

    class Meta:
        model = models.TaskWorker
        fields = ('id', 'task', 'worker', 'task_status', 'created_timestamp', 'last_updated',
                  'task_worker_results', 'worker_alias', 'task_worker_results_monitoring', 'updated_delta',
                  'requester_alias', 'module', 'project_name', 'task_with_data_and_results')
        read_only_fields = ('task', 'worker', 'created_timestamp', 'last_updated')

    def create(self, **kwargs):
        module = kwargs['module']
        module_instance = models.Module.objects.get(id=module)
        repetition = module_instance.repetition
        with self.lock:
            with transaction.atomic(): # select_for_update(nowait=False)
                tasks = models.Task.objects.filter(module=module).exclude(
                    task_workers__worker=kwargs['worker']) \
                    .annotate(task_worker_count=Count('task_workers')) \
                    .filter(module__repetition__gt=F('task_worker_count')).first()
                if not tasks:
                    tasks = models.Task.objects.filter(module=module) \
                        .exclude(task_workers__worker=kwargs['worker'], task_workers__task_status=6) \
                        .annotate(task_worker_count=Count('task_workers')) \
                        .filter(module__repetition__gt=F('task_worker_count')).first()
                if tasks:
                    task_worker = models.TaskWorker.objects.create(worker=kwargs['worker'], task=tasks)
                    tasks.status = 2
                    tasks.save()
                    return task_worker
                else:
                    raise ValidationError('No tasks left for this module')

    def get_worker_alias(self, obj):
        return obj.worker.alias

    def get_updated_delta(self, obj):
        from crowdsourcing.utils import get_time_delta

        return get_time_delta(obj.last_updated)

    def get_task_worker_results_monitoring(self, obj):
        task_worker_results = TaskWorkerResultSerializer(instance=obj.task_worker_results, many=True,
                                                         fields=('template_item_id', 'result')).data
        return task_worker_results

    def get_requester_alias(self, obj):
        return obj.task.module.owner.alias

    def get_module(self, obj):
        return {'id': obj.task.module.id, 'name': obj.task.module.name, 'price': obj.task.module.price}

    def get_project_name(self, obj):
        return obj.task.module.project.name

    def get_task_with_data_and_results(self, obj):
        task = TaskSerializer(instance=obj.task, fields=('id', 'task_template')).data
        template = task['task_template']
        task_worker_results = TaskWorkerResultSerializer(instance=obj.task_worker_results, many=True,
                                                            fields=('template_item_id', 'result')).data
        for task_worker_result in task_worker_results:
            for item in template['template_items']:
                if task_worker_result['template_item_id'] == item['id'] and item['role'] == 'input' \
                and task_worker_result['result'] is not None and \
                (item['type'] == 'text_field' or item['type'] == 'text_area'):
                    item['values'] = task_worker_result['result']
        template['template_items'] = sorted(template['template_items'], key=lambda k: k['position'])
        return template


class TaskSerializer(DynamicFieldsModelSerializer):
    task_workers = TaskWorkerSerializer(many=True, read_only=True)
    task_workers_monitoring = serializers.SerializerMethodField()
    task_template = serializers.SerializerMethodField()
    template_items_monitoring = serializers.SerializerMethodField()

    class Meta:
        model = models.Task
        fields = ('id', 'module', 'status', 'deleted', 'created_timestamp', 'last_updated', 'data',
                  'task_workers', 'task_workers_monitoring', 'task_template', 'template_items_monitoring')
        read_only_fields = ('created_timestamp', 'last_updated', 'deleted')

    def create(self, **kwargs):
        task = models.Task.objects.create(**self.validated_data)
        return task<commit_msg>returning answer as extra field<commit_after>from crowdsourcing import models
from rest_framework import serializers
from crowdsourcing.serializers.dynamic import DynamicFieldsModelSerializer
from crowdsourcing.serializers.template import TemplateItemSerializer
from rest_framework.exceptions import ValidationError
from django.db import transaction
from crowdsourcing.serializers.template import TemplateSerializer
import json
from django.db.models import Count, F


class TaskWorkerResultListSerializer(serializers.ListSerializer):
    def create(self, **kwargs):
        for item in self.validated_data:
            models.TaskWorkerResult.objects.get_or_create(task_worker=kwargs['task_worker'], **item)


class TaskWorkerResultSerializer(DynamicFieldsModelSerializer):
    template_item_id = serializers.SerializerMethodField()

    class Meta:
        model = models.TaskWorkerResult
        list_serializer_class = TaskWorkerResultListSerializer
        fields = ('id', 'template_item', 'result', 'status', 'created_timestamp', 'last_updated', 'template_item_id')
        read_only_fields = ('created_timestamp', 'last_updated')

    def create(self, **kwargs):
        models.TaskWorkerResult.objects.get_or_create(self.validated_data)

    def get_template_item_id(self, obj):
        template_item = TemplateItemSerializer(instance=obj.template_item).data
        return template_item['id']


class TaskWorkerSerializer(DynamicFieldsModelSerializer):
    import multiprocessing
    
    lock = multiprocessing.Lock()
    task_worker_results = TaskWorkerResultSerializer(many=True, read_only=True)
    worker_alias = serializers.SerializerMethodField()
    task_worker_results_monitoring = serializers.SerializerMethodField()
    updated_delta = serializers.SerializerMethodField()
    requester_alias = serializers.SerializerMethodField()
    module = serializers.SerializerMethodField()
    project_name = serializers.SerializerMethodField()
    task_with_data_and_results = serializers.SerializerMethodField()

    class Meta:
        model = models.TaskWorker
        fields = ('id', 'task', 'worker', 'task_status', 'created_timestamp', 'last_updated',
                  'task_worker_results', 'worker_alias', 'task_worker_results_monitoring', 'updated_delta',
                  'requester_alias', 'module', 'project_name', 'task_with_data_and_results')
        read_only_fields = ('task', 'worker', 'created_timestamp', 'last_updated')

    def create(self, **kwargs):
        module = kwargs['module']
        module_instance = models.Module.objects.get(id=module)
        repetition = module_instance.repetition
        with self.lock:
            with transaction.atomic(): # select_for_update(nowait=False)
                tasks = models.Task.objects.filter(module=module).exclude(
                    task_workers__worker=kwargs['worker']) \
                    .annotate(task_worker_count=Count('task_workers')) \
                    .filter(module__repetition__gt=F('task_worker_count')).first()
                if not tasks:
                    tasks = models.Task.objects.filter(module=module) \
                        .exclude(task_workers__worker=kwargs['worker'], task_workers__task_status=6) \
                        .annotate(task_worker_count=Count('task_workers')) \
                        .filter(module__repetition__gt=F('task_worker_count')).first()
                if tasks:
                    task_worker = models.TaskWorker.objects.create(worker=kwargs['worker'], task=tasks)
                    tasks.status = 2
                    tasks.save()
                    return task_worker
                else:
                    raise ValidationError('No tasks left for this module')

    def get_worker_alias(self, obj):
        return obj.worker.alias

    def get_updated_delta(self, obj):
        from crowdsourcing.utils import get_time_delta

        return get_time_delta(obj.last_updated)

    def get_task_worker_results_monitoring(self, obj):
        task_worker_results = TaskWorkerResultSerializer(instance=obj.task_worker_results, many=True,
                                                         fields=('template_item_id', 'result')).data
        return task_worker_results

    def get_requester_alias(self, obj):
        return obj.task.module.owner.alias

    def get_module(self, obj):
        return {'id': obj.task.module.id, 'name': obj.task.module.name, 'price': obj.task.module.price}

    def get_project_name(self, obj):
        return obj.task.module.project.name

    def get_task_with_data_and_results(self, obj):
        task = TaskSerializer(instance=obj.task, fields=('id', 'task_template')).data
        template = task['task_template']
        task_worker_results = TaskWorkerResultSerializer(instance=obj.task_worker_results, many=True,
                                                            fields=('template_item_id', 'result')).data
        for task_worker_result in task_worker_results:
            for item in template['template_items']:
                if task_worker_result['template_item_id'] == item['id'] and item['role'] == 'input' \
                and task_worker_result['result'] is not None:
                    item['answer'] = task_worker_result['result']
        template['template_items'] = sorted(template['template_items'], key=lambda k: k['position'])
        return template


class TaskSerializer(DynamicFieldsModelSerializer):
    task_workers = TaskWorkerSerializer(many=True, read_only=True)
    task_workers_monitoring = serializers.SerializerMethodField()
    task_template = serializers.SerializerMethodField()
    template_items_monitoring = serializers.SerializerMethodField()

    class Meta:
        model = models.Task
        fields = ('id', 'module', 'status', 'deleted', 'created_timestamp', 'last_updated', 'data',
                  'task_workers', 'task_workers_monitoring', 'task_template', 'template_items_monitoring')
        read_only_fields = ('created_timestamp', 'last_updated', 'deleted')

    def create(self, **kwargs):
        task = models.Task.objects.create(**self.validated_data)
        return task
{'fraction_non_alphanumeric': 0.09865434500648508, 'fraction_numerical': 0.00032425421530479895, 'mean_word_length': 3.2526714925887625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6698259', 'n_tokens_mistral': 3260, 'n_tokens_neox': 3162, 'n_words': 599}