<commit_before>package ch.uzh.ifi.pdeboer.pplib.hcomp.mturk

import ch.uzh.ifi.pdeboer.pplib.hcomp._
import ch.uzh.ifi.pdeboer.pplib.util.{GrowingTimer, LazyLogger, U}

import scala.concurrent.duration._
import scala.xml.NodeSeq

/**
 * Created by pdeboer on 19/11/14.
 */
class MTurkManager(val service: MTurkService, val query: HCompQuery, val properties: HCompQueryProperties) extends LazyLogger {
	var hit = ""
	var cancelled: Boolean = false

	def waitForResponse() = {
		val timer = new GrowingTimer(1 second, 1.0001, 20 seconds)
		U.retry(100000) {
			//at least 27h
			if (!cancelled) {
				timer.waitTime

				val answer = poll()
				if (answer == None) throw new IllegalStateException("let's wait some more")
				answer
			} else None
		}
	}

	def cancelHIT(): Unit = {
		cancelled = true
		try {
			service.DisableHIT(hit)
		}
		catch {
			case e: Throwable => logger.error(s"got exception when trying to cancel query '${query.question}'", e)
		}
		//service.deleteHITs(Array(hit.getHITId), true, true, emptyCallback)
	}

	/**
	 * @return HIT ID
	 */
	def createHIT(): String = {
		if (scala.xml.PCData(query.question).length > 1999)
			logger.error("your question was longer than 1999 characters, which is not allowed by MTurk. Truncated the question to " + query.question.take(1999))
		val mtQuery = MTQuery.convert(query)

		val ONE_DAY: Int = 60 * 60 * 24
		val dollars: Double = properties.paymentCents.toDouble / 100d

		val qualifications: List[QualificationRequirement] = List(
			//QualificationRequirement.Worker_Locale === "US",
			QualificationRequirement.Worker_NumberHITsApproved >= 1000)
		val hitTypeID = service.RegisterHITType(query.title.take(120), query.question, Price(dollars.toString), ONE_DAY, Seq.empty[String], ONE_DAY, qualifications)
		hit = service.CreateHIT(hitTypeID, new Question(mtQuery.xml), ONE_DAY, 1).HITId
		hit
	}

	def poll(): Option[HCompAnswer] = {
		logger.debug("checking for answer..")
		val assignments = service.GetAssignmentsForHIT(hit)
		assignments.headOption match {
			case None => None
			case Some(a: Assignment) => handleAssignmentResult(a)
		}
	}

	def handleAssignmentResult(a: Assignment): Option[HCompAnswer] = {
		try {
			//We approve all assignments by default. Don't like rejections
			service.ApproveAssignment(a)
		}
		catch {
			case e: Exception => logger.error("could not approve assignment", e)
		}

		val xml = a.AnswerXML
		val answer = MTQuery.convert(query).interpret(xml)
		answer.map(answer => {<commit_msg>worker locale<commit_after>package ch.uzh.ifi.pdeboer.pplib.hcomp.mturk

import ch.uzh.ifi.pdeboer.pplib.hcomp._
import ch.uzh.ifi.pdeboer.pplib.util.{GrowingTimer, LazyLogger, U}

import scala.concurrent.duration._
import scala.xml.NodeSeq

/**
 * Created by pdeboer on 19/11/14.
 */
class MTurkManager(val service: MTurkService, val query: HCompQuery, val properties: HCompQueryProperties) extends LazyLogger {
	var hit = ""
	var cancelled: Boolean = false

	def waitForResponse() = {
		val timer = new GrowingTimer(1 second, 1.0001, 20 seconds)
		U.retry(100000) {
			//at least 27h
			if (!cancelled) {
				timer.waitTime

				val answer = poll()
				if (answer == None) throw new IllegalStateException("let's wait some more")
				answer
			} else None
		}
	}

	def cancelHIT(): Unit = {
		cancelled = true
		try {
			service.DisableHIT(hit)
		}
		catch {
			case e: Throwable => logger.error(s"got exception when trying to cancel query '${query.question}'", e)
		}
		//service.deleteHITs(Array(hit.getHITId), true, true, emptyCallback)
	}

	/**
	 * @return HIT ID
	 */
	def createHIT(): String = {
		if (scala.xml.PCData(query.question).length > 1999)
			logger.error("your question was longer than 1999 characters, which is not allowed by MTurk. Truncated the question to " + query.question.take(1999))
		val mtQuery = MTQuery.convert(query)

		val ONE_DAY: Int = 60 * 60 * 24
		val dollars: Double = properties.paymentCents.toDouble / 100d

		val qualifications: List[QualificationRequirement] = List(
			QualificationRequirement.Worker_Locale === "US",
			QualificationRequirement.Worker_NumberHITsApproved >= 1000)
		val hitTypeID = service.RegisterHITType(query.title.take(120), query.question, Price(dollars.toString), ONE_DAY, Seq.empty[String], ONE_DAY, qualifications)
		hit = service.CreateHIT(hitTypeID, new Question(mtQuery.xml), ONE_DAY, 1).HITId
		hit
	}

	def poll(): Option[HCompAnswer] = {
		logger.debug("checking for answer..")
		val assignments = service.GetAssignmentsForHIT(hit)
		assignments.headOption match {
			case None => None
			case Some(a: Assignment) => handleAssignmentResult(a)
		}
	}

	def handleAssignmentResult(a: Assignment): Option[HCompAnswer] = {
		try {
			//We approve all assignments by default. Don't like rejections
			service.ApproveAssignment(a)
		}
		catch {
			case e: Exception => logger.error("could not approve assignment", e)
		}

		val xml = a.AnswerXML
		val answer = MTQuery.convert(query).interpret(xml)
		answer.map(answer => {
{'fraction_non_alphanumeric': 0.11918138041733548, 'fraction_numerical': 0.020465489566613163, 'mean_word_length': 4.076374745417516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6080515', 'n_tokens_mistral': 1913, 'n_tokens_neox': 1646, 'n_words': 411}