<commit_before>import datetime
from decimal import Decimal

from boto.mturk.connection import MTurkConnection, MTurkRequestError
from boto.mturk.price import Price
from boto.mturk.qualification import (NumberHitsApprovedRequirement,
                                      PercentAssignmentsApprovedRequirement,
                                      Qualifications)
from boto.mturk.question import ExternalQuestion
from django.db.models import Q
from django.db import connection
from django.utils import timezone
from hashids import Hashids

from crowdsourcing.models import Task, TaskWorker, Rating
from csp import settings
from mturk.models import MTurkHIT, MTurkHITType, MTurkQualification, MTurkWorkerQualification
from mturk.utils import MultiLocaleRequirement, BoomerangRequirement

FLAG_Q_LOCALE = 0x1
FLAG_Q_RATE = 0x2
FLAG_Q_HITS = 0x4
FLAG_Q_BOOMERANG = 0x8
FLAG_Q_OTHER = 0x10

OP_LT = 'LessThan'
OP_LTEQ = 'LessThanOrEqualTo'
OP_GT = 'GreaterThan'
OP_GTEQ = 'GreaterThanOrEqualTo'
OP_EQ = 'EqualTo'
OP_NOT_EQ = 'NotEqualTo'
OP_E = 'Exists'
OP_DNE = 'DoesNotExist'
OP_IN = 'In'
OP_NOT_IN = 'NotIn'

WAIT_LIST_BUCKETS = [(1.80, 1.98), (1.60, 1.79), (1.40, 1.59), (1.20, 1.39), (1.0, 1.19)]

BOOMERANG_QUAL_INITIAL = 300


class MTurkProvider(object):
    description = 'This is a task authored by a requester on Daemo, a research crowdsourcing platform. ' \
                  'Mechanical Turk workers are welcome to do it'
    keywords = ['daemo']
    countries = ['US', 'CA']
    min_hits = 1000

    def __init__(self, host, aws_access_key_id, aws_secret_access_key):
        self.host = host
        self.connection = MTurkConnection(
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key,
            host=settings.MTURK_HOST
        )
        self.connection.APIVersion = "2014-08-15"
        if not self.host:
            raise ValueError("Please provide a host url")

    def get_connection(self):
        return self.connection

    def get_qualifications(self, owner_id, boomerang_threshold, project_group, add_boomerang):
        requirements = []
        approved_hits = NumberHitsApprovedRequirement('GreaterThan', self.min_hits)
        percentage_approved = PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 97)
        locale = MultiLocaleRequirement('In', self.countries)
        boomerang_qual, success = self.create_qualification_type(owner_id=owner_id,
                                                                 name='Boomerang Score #{}'.format(project_group),
                                                                 flag=FLAG_Q_BOOMERANG,
                                                                 description='No description available')
        boomerang = None
        if boomerang_threshold <= int(settings.BOOMERANG_MIDPOINT * 100):
            for i, bucket in enumerate(WAIT_LIST_BUCKETS):
                if bucket[1] <= boomerang_threshold:
                    boomerang_blacklist, success = \
                        self.create_qualification_type(owner_id=owner_id,
                                                       name='Boomerang Waitlist #{}-{}'.format(project_group, len(
                                                           WAIT_LIST_BUCKETS) + 1 - i),
                                                       flag=FLAG_Q_BOOMERANG,
                                                       description='No description available',
                                                       deny=True,
                                                       bucket=bucket)
                    boomerang = BoomerangRequirement(qualification_type_id=boomerang_blacklist.type_id,
                                                     comparator=OP_DNE,
                                                     integer_value=None)
                    if success and add_boomerang > 0:
                        requirements.append(boomerang)

        else:
            boomerang = BoomerangRequirement(qualification_type_id=boomerang_qual.type_id, comparator=OP_GTEQ,
                                             integer_value=boomerang_threshold)
            if success and add_boomerang > 0:
                requirements.append(boomerang)
        requirements.append(locale)
        if str(settings.MTURK_SYS_QUALIFICATIONS) == 'True':
            requirements.append(approved_hits)
            requirements.append(percentage_approved)

        return Qualifications(requirements), boomerang_qual

    def create_hits(self, project, tasks=None, repetition=None):
        # if project.min_rating > 0:
        #     return 'NOOP'
        if not tasks:
            # noinspection SqlResolve
            query = '''
                SELECT t.id, count(tw.id) worker_count FROM
                crowdsourcing_task t
                LEFT OUTER JOIN crowdsourcing_taskworker tw ON t.id = tw.task_id AND tw.status
                NOT IN (%(skipped)s, %(rejected)s, %(expired)s)
                WHERE project_id = %(project_id)s
                GROUP BY t.id
            '''
            tasks = Task.objects.raw(query, params={'skipped': TaskWorker.STATUS_SKIPPED,
                                                    'rejected': TaskWorker.STATUS_REJECTED,
                                                    'expired': TaskWorker.STATUS_EXPIRED,
                                                    'project_id': project.id})

        max_assignments = project.repetition
        # if str(settings.MTURK_ONLY) == 'True':
        #     max_assignments = project.repetition
        # else:
        #     max_assignments = 1

        qualifications = None
        # if str(settings.MTURK_QUALIFICATIONS) == 'True':
        rated_workers = Rating.objects.filter(origin_id=project.owner_id, origin_type=Rating.RATING_REQUESTER).count()
        add_boomerang = rated_workers > 0
        qualifications, boomerang_qual = self.get_qualifications(owner_id=project.owner_id,
                                                                 boomerang_threshold=int(project.min_rating * 100),
                                                                 project_group=project.group_id,
                                                                 add_boomerang=add_boomerang)
        duration = datetime.timedelta(
            minutes=project.task_time) if project.task_time is not None else datetime.timedelta(days=7)
        lifetime = project.deadline - timezone.now() if project.deadline is not None else datetime.timedelta(
            days=7)
        qualifications_mask = 0<commit_msg>rated_workers global<commit_after>import datetime
from decimal import Decimal

from boto.mturk.connection import MTurkConnection, MTurkRequestError
from boto.mturk.price import Price
from boto.mturk.qualification import (NumberHitsApprovedRequirement,
                                      PercentAssignmentsApprovedRequirement,
                                      Qualifications)
from boto.mturk.question import ExternalQuestion
from django.db.models import Q
from django.db import connection
from django.utils import timezone
from hashids import Hashids

from crowdsourcing.models import Task, TaskWorker, Rating
from csp import settings
from mturk.models import MTurkHIT, MTurkHITType, MTurkQualification, MTurkWorkerQualification
from mturk.utils import MultiLocaleRequirement, BoomerangRequirement

FLAG_Q_LOCALE = 0x1
FLAG_Q_RATE = 0x2
FLAG_Q_HITS = 0x4
FLAG_Q_BOOMERANG = 0x8
FLAG_Q_OTHER = 0x10

OP_LT = 'LessThan'
OP_LTEQ = 'LessThanOrEqualTo'
OP_GT = 'GreaterThan'
OP_GTEQ = 'GreaterThanOrEqualTo'
OP_EQ = 'EqualTo'
OP_NOT_EQ = 'NotEqualTo'
OP_E = 'Exists'
OP_DNE = 'DoesNotExist'
OP_IN = 'In'
OP_NOT_IN = 'NotIn'

WAIT_LIST_BUCKETS = [(1.80, 1.98), (1.60, 1.79), (1.40, 1.59), (1.20, 1.39), (1.0, 1.19)]

BOOMERANG_QUAL_INITIAL = 300


class MTurkProvider(object):
    description = 'This is a task authored by a requester on Daemo, a research crowdsourcing platform. ' \
                  'Mechanical Turk workers are welcome to do it'
    keywords = ['daemo']
    countries = ['US', 'CA']
    min_hits = 1000

    def __init__(self, host, aws_access_key_id, aws_secret_access_key):
        self.host = host
        self.connection = MTurkConnection(
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key,
            host=settings.MTURK_HOST
        )
        self.connection.APIVersion = "2014-08-15"
        if not self.host:
            raise ValueError("Please provide a host url")

    def get_connection(self):
        return self.connection

    def get_qualifications(self, owner_id, boomerang_threshold, project_group, add_boomerang):
        requirements = []
        approved_hits = NumberHitsApprovedRequirement('GreaterThan', self.min_hits)
        percentage_approved = PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 97)
        locale = MultiLocaleRequirement('In', self.countries)
        boomerang_qual, success = self.create_qualification_type(owner_id=owner_id,
                                                                 name='Boomerang Score #{}'.format(project_group),
                                                                 flag=FLAG_Q_BOOMERANG,
                                                                 description='No description available')
        boomerang = None
        if boomerang_threshold <= int(settings.BOOMERANG_MIDPOINT * 100):
            for i, bucket in enumerate(WAIT_LIST_BUCKETS):
                if bucket[1] <= boomerang_threshold:
                    boomerang_blacklist, success = \
                        self.create_qualification_type(owner_id=owner_id,
                                                       name='Boomerang Waitlist #{}-{}'.format(project_group, len(
                                                           WAIT_LIST_BUCKETS) + 1 - i),
                                                       flag=FLAG_Q_BOOMERANG,
                                                       description='No description available',
                                                       deny=True,
                                                       bucket=bucket)
                    boomerang = BoomerangRequirement(qualification_type_id=boomerang_blacklist.type_id,
                                                     comparator=OP_DNE,
                                                     integer_value=None)
                    if success and add_boomerang > 0:
                        requirements.append(boomerang)

        else:
            boomerang = BoomerangRequirement(qualification_type_id=boomerang_qual.type_id, comparator=OP_GTEQ,
                                             integer_value=boomerang_threshold)
            if success and add_boomerang > 0:
                requirements.append(boomerang)
        requirements.append(locale)
        if str(settings.MTURK_SYS_QUALIFICATIONS) == 'True':
            requirements.append(approved_hits)
            requirements.append(percentage_approved)

        return Qualifications(requirements), boomerang_qual

    def create_hits(self, project, tasks=None, repetition=None):
        # if project.min_rating > 0:
        #     return 'NOOP'
        if not tasks:
            # noinspection SqlResolve
            query = '''
                SELECT t.id, count(tw.id) worker_count FROM
                crowdsourcing_task t
                LEFT OUTER JOIN crowdsourcing_taskworker tw ON t.id = tw.task_id AND tw.status
                NOT IN (%(skipped)s, %(rejected)s, %(expired)s)
                WHERE project_id = %(project_id)s
                GROUP BY t.id
            '''
            tasks = Task.objects.raw(query, params={'skipped': TaskWorker.STATUS_SKIPPED,
                                                    'rejected': TaskWorker.STATUS_REJECTED,
                                                    'expired': TaskWorker.STATUS_EXPIRED,
                                                    'project_id': project.id})

        max_assignments = project.repetition
        # if str(settings.MTURK_ONLY) == 'True':
        #     max_assignments = project.repetition
        # else:
        #     max_assignments = 1

        qualifications = None
        # if str(settings.MTURK_QUALIFICATIONS) == 'True':
        rated_workers = Rating.objects.filter(origin_type=Rating.RATING_REQUESTER).count()
        add_boomerang = rated_workers > 0
        qualifications, boomerang_qual = self.get_qualifications(owner_id=project.owner_id,
                                                                 boomerang_threshold=int(project.min_rating * 100),
                                                                 project_group=project.group_id,
                                                                 add_boomerang=add_boomerang)
        duration = datetime.timedelta(
            minutes=project.task_time) if project.task_time is not None else datetime.timedelta(days=7)
        lifetime = project.deadline - timezone.now() if project.deadline is not None else datetime.timedelta(
            days=7)
        qualifications_mask = 0
{'fraction_non_alphanumeric': 0.07159668969706173, 'fraction_numerical': 0.011084959380457065, 'mean_word_length': 1.761425576519916, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 76, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2179678', 'n_tokens_mistral': 3578, 'n_tokens_neox': 3409, 'n_words': 711}