<commit_before>#This script makes it easy to create and delete new qualification types in MTurk.
#Please note that deleting qualification types is c
# http://opensource.org/licenses/MIT

#===================== IMPORTS --- SETUP --- GLOBAL VARS ======================
import os
import sys
import re

#import stuff from boto
from boto.mturk.question import *
from boto.mturk.connection import MTurkConnection
from argparse import ArgumentParser

#================================= __MAIN__ ===================================


def main():
    # define some strings for ArgParser
    progr_descr = 'This program loads native qualification tests into MTurk.'
    lang_help = 'This specifies the folder from which to read questions and answers.'
    account_help = 'This specifies the folder from which to read credential information.'
# set up argument parser
    arg_parser = ArgumentParser(description=progr_descr)
    arg_parser.add_argument('language', help=lang_help)
    arg_parser.add_argument('account', help=account_help)
    cmd_arg = arg_parser.parse_args()

# set up language directory
    root = os.getcwd()
    if cmd_arg.language not in os.listdir(root):
        raise MissingFolderException(cmd_arg.language)

    lang_root = os.path.join(root, cmd_arg.language)

# load properties file
    properties_f_name = find_file('properties', lang_root)
    properties = read_settings_file(properties_f_name)

# load question and answer src files
    question_src = find_file('questions', lang_root)
    question_xml = parse_question_file(question_src)
    answer_src = find_file('answers', lang_root)
    answer_xml = parse_answer_file(answer_src)

    print question_xml
# set up a connection.
#     connection = create_mturk_connection(os.path.join(root, cmd_arg.account))

# # create qualification test
#     connection.create_qualification_type(name=properties['name'],
#                                          description=properties['description'],
#                                          test=question_xml,
#                                          answer_key=answer_xml,
#                                          status='Active',
#                                          keywords=properties['keywords'],
#                                          retry_delay=properties['retrydelayinseconds'],
#                                          test_duration=properties['testdurationinseconds'])


class AnswerKey(ValidatingXML, list):
    schema_url = ('http://mechanicalturk.amazonaws.com/'
        'AWSMechanicalTurkDataSchemas/2005-10-01/AnswerKey.xsd')
    xml_template = ('<?xml version="1.0" encoding="UTF-8"?>\n'
        '<AnswerKey xmlns="{schema_url}">\n'
        '{answers}\n'
        '</AnswerKey>\n')

    def get_as_xml(self):
        items = '\n'.join(item.get_as_xml() for item in self)
        return self.xml_template.format(schema_url=self.schema_url, 
            answers=items)


class AnswerOption():
    def __init__(self, question_IDs, score):
        self.question_IDs = question_IDs
        self.score = score
        self.template = ('<AnswerOption>'
            '{options}\n'
            '</AnswerOption>\n')

    def get_as_xml(self):
        IDs = [SimpleField('SelectionIdentifier', ID)
               for ID in self.question_IDs]
        fields = IDs + [SimpleField('AnswerScore', self.score)]
        fields_combined = '\n'.join(f.get_as_xml() for f in fields)
        return self.template.format(options=fields_combined)


class QuestionAnswer(Question):
    template = '<Question>{0}\n{1}</Question>'

    def __init__(self, identifier, options):
        self.identifier = SimpleField('QuestionIdentifier', identifier)
        #self.answer_options = [opt.get_as_xml() for opt in options]
        self.answer_options = options

    def get_as_xml(self):
        return self.template.format(self.identifier.get_as_xml(),
                                    self.answer_options.get_as_xml())


def parse_answer_file(file_name):
    q_rgx = re.compile('Question (.*)')
    score_rgx = re.compile('Score (.*)')
    ans_rgx = re.compile('(Answer.+?)(?=Score)', flags=re.DOTALL)
    a_rgx = re.compile('Answer (.+)')
    with open(file_name, 'rU') as ans_file:
        ans_file_str = ans_file.read()
        questions = q_rgx.findall(ans_file_str)
        scores = score_rgx.findall(ans_file_str)
        answers = ans_rgx.findall(ans_file_str)
        assert len(questions) == len(answers) == len(scores)
    question_list = []
    for indx in xrange(len(questions)):
        q_answers = a_rgx.findall(answers[indx])
        answer_opts = AnswerOption(q_answers, scores[indx])
        question_list.append(QuestionAnswer(questions[indx], answer_opts))
    return AnswerKey(question_list)


def parse_question_file(file_name):<commit_msg>fixed var names and strings in boto-based classes<commit_after>#This script makes it easy to create and delete new qualification types in MTurk.
#Please note that deleting qualification types is c
# http://opensource.org/licenses/MIT

#===================== IMPORTS --- SETUP --- GLOBAL VARS ======================
import os
import sys
import re

#import stuff from boto
from boto.mturk.question import *
from boto.mturk.connection import MTurkConnection
from argparse import ArgumentParser

#================================= __MAIN__ ===================================


def main():
    # define some strings for ArgParser
    progr_descr = 'This program loads native qualification tests into MTurk.'
    lang_help = 'This specifies the folder from which to read questions and answers.'
    account_help = 'This specifies the folder from which to read credential information.'
# set up argument parser
    arg_parser = ArgumentParser(description=progr_descr)
    arg_parser.add_argument('language', help=lang_help)
    arg_parser.add_argument('account', help=account_help)
    cmd_arg = arg_parser.parse_args()

# set up language directory
    root = os.getcwd()
    if cmd_arg.language not in os.listdir(root):
        raise MissingFolderException(cmd_arg.language)

    lang_root = os.path.join(root, cmd_arg.language)

# load properties file
    properties_f_name = find_file('properties', lang_root)
    properties = read_settings_file(properties_f_name)

# load question and answer src files
    question_src = find_file('questions', lang_root)
    question_xml = parse_question_file(question_src)
    answer_src = find_file('answers', lang_root)
    answer_xml = parse_answer_file(answer_src)

    print question_xml
# set up a connection.
#     connection = create_mturk_connection(os.path.join(root, cmd_arg.account))

# # create qualification test
#     connection.create_qualification_type(name=properties['name'],
#                                          description=properties['description'],
#                                          test=question_xml,
#                                          answer_key=answer_xml,
#                                          status='Active',
#                                          keywords=properties['keywords'],
#                                          retry_delay=properties['retrydelayinseconds'],
#                                          test_duration=properties['testdurationinseconds'])


class AnswerKey(ValidatingXML, list):
    schema_url = ('http://mechanicalturk.amazonaws.com/'
        'AWSMechanicalTurkDataSchemas/2005-10-01/AnswerKey.xsd')
    xml_template = ('<?xml version="1.0" encoding="UTF-8"?>\n'
        '<AnswerKey xmlns="{schema_url}">\n'
        '{answers}\n'
        '</AnswerKey>\n')

    def get_as_xml(self):
        items = '\n'.join(item.get_as_xml() for item in self)
        return self.xml_template.format(schema_url=self.schema_url, 
            answers=items)


class AnswerOption():
    def __init__(self, correct_ans, score):
        self.correct_ans = correct_ans
        self.score = score
        self.template = ('<AnswerOption>\n'
            '{options}\n'
            '</AnswerOption>')

    def get_as_xml(self):
        IDs = [SimpleField('SelectionIdentifier', ID)
               for ID in self.correct_ans]
        fields = IDs + [SimpleField('AnswerScore', self.score)]
        fields_combined = '\n'.join(f.get_as_xml() for f in fields)
        return self.template.format(options=fields_combined)


class QuestionAnswer(Question):
    template = '<Question>\n{0}\n{1}\n</Question>'

    def __init__(self, identifier, options):
        self.identifier = SimpleField('QuestionIdentifier', identifier)
        #self.answer_options = [opt.get_as_xml() for opt in options]
        self.answer_options = options

    def get_as_xml(self):
        return self.template.format(self.identifier.get_as_xml(),
                                    self.answer_options.get_as_xml())


def parse_answer_file(file_name):
    q_rgx = re.compile('Question (.*)')
    score_rgx = re.compile('Score (.*)')
    ans_rgx = re.compile('(Answer.+?)(?=Score)', flags=re.DOTALL)
    a_rgx = re.compile('Answer (.+)')
    with open(file_name, 'rU') as ans_file:
        ans_file_str = ans_file.read()
        questions = q_rgx.findall(ans_file_str)
        scores = score_rgx.findall(ans_file_str)
        answers = ans_rgx.findall(ans_file_str)
        assert len(questions) == len(answers) == len(scores)
    question_list = []
    for indx in xrange(len(questions)):
        q_answers = a_rgx.findall(answers[indx])
        answer_opts = AnswerOption(q_answers, scores[indx])
        question_list.append(QuestionAnswer(questions[indx], answer_opts))
    return AnswerKey(question_list)


def parse_question_file(file_name):
{'fraction_non_alphanumeric': 0.12703209670696122, 'fraction_numerical': 0.0027094622759483117, 'mean_word_length': 2.9607924061081303, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 2, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 34}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2080032', 'n_tokens_mistral': 2808, 'n_tokens_neox': 2689, 'n_words': 582}