<commit_before>__author__ = '<EMAIL>'
__author__ = '<EMAIL>'

import datetime
import logging
import random
import traceback

from models import transforms
from models.models import Student
from models.models import EventEntity
from models import utils as models_utils
from models import jobs
from models import event_transforms

from models.models import QuestionDAO
from models.models import QuestionGroupDAO
from models.models import MemcacheManager

from models.progress import UnitLessonCompletionTracker

class ActivityScoreParser(jobs.MapReduceJob):
    """
        Class to parse the data returned from query of Event activities.

        An tag-assessment Event is recorded every time a student attempts
        a GCB question or Quizly exercise.  The only way to determine
        student performance on questions and exercises is to
        query the Events database, extract tag-assessment events and
        process them.
    """

    CUTOFF_DATE = datetime.datetime(2016,8,1)


    def __init__(self):
        """holds activity score info unit -> lesson -> question"""
        self.activity_scores = { }
        self.params = {}
        self.num_attempts_dict = { }
        self.quizly_desc = {               # instance_id:description
            'LXgF4NO50hNM':{'desc':'Pause the Player','name':'quiz_pause_the_player'},       # Unit 2
            'BtQ8hSoGkeml':{'desc':'Stop the Player','name':'quiz_button_click_stop_player'},
            'Dstsv7VuDQb5':{'desc':'Stop Player if playing','name':'quiz_stop_player_if_playing'},
            'twxBgieSEwqs':{'desc':'If/else stop/start Player','name':'quiz_if_else_stop_start_player'},
            'a3uBZXYSOJee':{'desc':'Set background color','name':'quiz_background_color'},   # Unit 3
            'pnhvzarYPPW1':{'desc':'Set text color','name':'quiz_text_color'},
            'G3qzTftPYKTe':{'desc':'Increment a variable','name':'quiz_incrementing'},
            '4kITN7u5hdsO':{'desc':'Initialize global variable','name':'quiz_initialize_variable'},
            'pCZugPUxlHeb':{'desc':'Initializing','name':'quiz_incrementing'},
            '8T30OkUf5r1r':{'desc':'Simple if/else','name':'quiz_simple_if_else'},
            'KQctST8skmaC':{'desc':'Procedure to double a variable','name':'quiz_proc_double'},
            'v2m4Ks25S1MX':{'desc':'Procedure to add globals','name':'quiz_add_global'},
            'rCgLbJRceEbn':{'desc':'Procedure to reset the score','name':'quiz_reset_score'},     # Unit 4   
            '7uowepixSjT4':{'desc':'Procedure to calculate the hit rate','name':'quiz_calculate_hit_rate'},
            'w18q4UWKxvlM':{'desc':'Fix a bug in updateScore procedure','name':'quiz_procedure_bug'},
            'rvjUJMaLZ56s':{'desc':'If/else greater than','name':'quiz_if_x_greater_than_y'},
        }

    @staticmethod
    def get_description():
        return 'activity answers parser'

    @staticmethod
    def entity_class():
        return EventEntity

    def get_num_attempts_dict():
        return _num_attempts_dict

    @classmethod
    def _get_questions_by_question_id(cls, questions_by_usage_id):
        ''' Retrieves every question in the course returning 
            them in a dict:  { id:questionDAO, ... }

            @param questions_by_usage_id.values() is a dict:
             {unit, lesson, sequence, weight, quid}
        '''
        ret = {}
        ret['single'] = {}
        ret['grouped'] = {}
        for question in questions_by_usage_id.values():
            question_single = QuestionDAO.load(question['id'])
            if question_single:
                ret['single'][question['id']] = question_single
            else:
                question_group = QuestionGroupDAO.load(question['id'])
                if question_group:
                    ret['grouped'][question['id']] = {}
                    for item in question_group.items:
                        ret['grouped'][question['id']][item['question']] = QuestionDAO.load(item['question'])
        return ret

    def build_additional_mapper_params(self, app_context):
        questions_by_usage_id = event_transforms.get_questions_by_usage_id(app_context)
        return {
            'questions_by_usage_id':
                questions_by_usage_id,
            'valid_question_ids': (
                event_transforms.get_valid_question_ids()),
            'group_to_questions': (
                event_transforms.get_group_to_questions()),
            'assessment_weights':
                event_transforms.get_assessment_weights(app_context),
            'unscored_lesson_ids':
                event_transforms.get_unscored_lesson_ids(app_context),
            'questions_by_question_id':
                ActivityScoreParser._get_questions_by_question_id(questions_by_usage_id)
            }

    def parse_quizly_scores(self, data, instance_id, timestamp, student, student_answers):
         """
             Processes Quizly exercises extracted from the Events query using the
             instance_id for the quid.

             Because Quizly exercises are not officially questions -- something that
             needs to be fixed -- they don't have a quid and need to be processed in
             an ad-hoc way. There is no question_info dict that can be used to get
             information about Quizly exercises.

             Also Quizly exercises don't have a 'sequence' number within the lesson.
             To get around that we assign random numbers starting at 10. One problem
             with this approach is that it doesn't preserve the sequence of Quizly
             exercises within the lesson.
         """

#        logging.debug('***RAM*** A question with instance_id = ' + str(instance_id) +
#          ' and no quid and location = ' + str(data['location']))
         url = data['location']
         quizly_unit_id =  int(url[url.find('unit=') + len('unit=') : url.find('&lesson=')])
         quizly_lesson_id = int(url[ url.find('&lesson=') + len('&lesson=') : ])
         quizly_score = data['score']
         quizly_answer = data['answer']
         quizly_type = 'Quizly'      # reported in data as SaQuestion
         quizly_sequence = random.randint(10,30)                  #### Make up a random sequence #

         # Create a dict for this Quizly exercise
         question_answer_dict = {}
         question_answer_dict['unit_id'] = quizly_unit_id
         question_answer_dict['lesson_id'] = quizly_lesson_id
         question_answer_dict['sequence'] = quizly_sequence       # Not given
         question_answer_dict['question_id'] = instance_id        # Use instance_id as quid
         if instance_id in self.quizly_desc:
             question_answer_dict['description'] = self.quizly_desc[instance_id]
         else:
             question_answer_dict['description'] = 'Quizly ' + instance_id
         question_answer_dict['question_type'] = quizly_type
         question_answer_dict['timestamp'] = timestamp
         question_answer_dict['answers'] = [quizly_answer]   # T or F
         question_answer_dict['score'] = quizly_score
         question_answer_dict['weighted_score'] = quizly_score
         question_answer_dict['tallied'] = False
         question_answer_dict['choices'] = [{'text':'T','score':1},{'text':'F','score': 0}]   # Quizly's are T/F

         # Use the instance_id to count the number of attempts for each Quizly exercise by a given student
         if not student.email in self.num_attempts_dict:
             self.num_attempts_dict[student.email] = {}
         if not instance_id in self.num_attempts_dict[student.email]:
             self.num_attempts_dict[student.email][instance_id] = 1
         else:<commit_msg>Possibly unnecessary hack to remove error on bogus Event record.<commit_after>__author__ = '<EMAIL>'
__author__ = '<EMAIL>'

import datetime
import logging
import random
import traceback

from models import transforms
from models.models import Student
from models.models import EventEntity
from models import utils as models_utils
from models import jobs
from models import event_transforms

from models.models import QuestionDAO
from models.models import QuestionGroupDAO
from models.models import MemcacheManager

from models.progress import UnitLessonCompletionTracker

class ActivityScoreParser(jobs.MapReduceJob):
    """
        Class to parse the data returned from query of Event activities.

        An tag-assessment Event is recorded every time a student attempts
        a GCB question or Quizly exercise.  The only way to determine
        student performance on questions and exercises is to
        query the Events database, extract tag-assessment events and
        process them.
    """

    CUTOFF_DATE = datetime.datetime(2016,8,1)


    def __init__(self):
        """holds activity score info unit -> lesson -> question"""
        self.activity_scores = { }
        self.params = {}
        self.num_attempts_dict = { }
        self.quizly_desc = {               # instance_id:description
            'LXgF4NO50hNM':{'desc':'Pause the Player','name':'quiz_pause_the_player'},       # Unit 2
            'BtQ8hSoGkeml':{'desc':'Stop the Player','name':'quiz_button_click_stop_player'},
            'Dstsv7VuDQb5':{'desc':'Stop Player if playing','name':'quiz_stop_player_if_playing'},
            'twxBgieSEwqs':{'desc':'If/else stop/start Player','name':'quiz_if_else_stop_start_player'},
            'a3uBZXYSOJee':{'desc':'Set background color','name':'quiz_background_color'},   # Unit 3
            'pnhvzarYPPW1':{'desc':'Set text color','name':'quiz_text_color'},
            'G3qzTftPYKTe':{'desc':'Increment a variable','name':'quiz_incrementing'},
            '4kITN7u5hdsO':{'desc':'Initialize global variable','name':'quiz_initialize_variable'},
            'pCZugPUxlHeb':{'desc':'Initializing','name':'quiz_incrementing'},
            '8T30OkUf5r1r':{'desc':'Simple if/else','name':'quiz_simple_if_else'},
            'KQctST8skmaC':{'desc':'Procedure to double a variable','name':'quiz_proc_double'},
            'v2m4Ks25S1MX':{'desc':'Procedure to add globals','name':'quiz_add_global'},
            'rCgLbJRceEbn':{'desc':'Procedure to reset the score','name':'quiz_reset_score'},     # Unit 4   
            '7uowepixSjT4':{'desc':'Procedure to calculate the hit rate','name':'quiz_calculate_hit_rate'},
            'w18q4UWKxvlM':{'desc':'Fix a bug in updateScore procedure','name':'quiz_procedure_bug'},
            'rvjUJMaLZ56s':{'desc':'If/else greater than','name':'quiz_if_x_greater_than_y'},
        }

    @staticmethod
    def get_description():
        return 'activity answers parser'

    @staticmethod
    def entity_class():
        return EventEntity

    def get_num_attempts_dict():
        return _num_attempts_dict

    @classmethod
    def _get_questions_by_question_id(cls, questions_by_usage_id):
        ''' Retrieves every question in the course returning 
            them in a dict:  { id:questionDAO, ... }

            @param questions_by_usage_id.values() is a dict:
             {unit, lesson, sequence, weight, quid}
        '''
        ret = {}
        ret['single'] = {}
        ret['grouped'] = {}
        for question in questions_by_usage_id.values():
            question_single = QuestionDAO.load(question['id'])
            if question_single:
                ret['single'][question['id']] = question_single
            else:
                question_group = QuestionGroupDAO.load(question['id'])
                if question_group:
                    ret['grouped'][question['id']] = {}
                    for item in question_group.items:
                        ret['grouped'][question['id']][item['question']] = QuestionDAO.load(item['question'])
        return ret

    def build_additional_mapper_params(self, app_context):
        questions_by_usage_id = event_transforms.get_questions_by_usage_id(app_context)
        return {
            'questions_by_usage_id':
                questions_by_usage_id,
            'valid_question_ids': (
                event_transforms.get_valid_question_ids()),
            'group_to_questions': (
                event_transforms.get_group_to_questions()),
            'assessment_weights':
                event_transforms.get_assessment_weights(app_context),
            'unscored_lesson_ids':
                event_transforms.get_unscored_lesson_ids(app_context),
            'questions_by_question_id':
                ActivityScoreParser._get_questions_by_question_id(questions_by_usage_id)
            }

    def parse_quizly_scores(self, data, instance_id, timestamp, student, student_answers):
         """
             Processes Quizly exercises extracted from the Events query using the
             instance_id for the quid.

             Because Quizly exercises are not officially questions -- something that
             needs to be fixed -- they don't have a quid and need to be processed in
             an ad-hoc way. There is no question_info dict that can be used to get
             information about Quizly exercises.

             Also Quizly exercises don't have a 'sequence' number within the lesson.
             To get around that we assign random numbers starting at 10. One problem
             with this approach is that it doesn't preserve the sequence of Quizly
             exercises within the lesson.
         """

#        logging.debug('***RAM*** A question with instance_id = ' + str(instance_id) +
#          ' and no quid and location = ' + str(data['location']))
         url = data['location']
         quizly_unit_id =  int(url[url.find('unit=') + len('unit=') : url.find('&lesson=')])
         quizly_lesson_id = int(url[ url.find('&lesson=') + len('&lesson=') : ])
         quizly_score = data['score']
#          if instance_id == 'rCgLbJRceEbn' or instance_id == '7uowepixSjT4':
#              logging.warning('***RAM*** rCgLbJRceEbn data = ' + str(data))

         if 'answer' in data:           # Takes care of legacy events that are missing answer?
             quizly_answer = data['answer']
         else:
             quizly_answer = False

         quizly_type = 'Quizly'      # reported in data as SaQuestion
         quizly_sequence = random.randint(10,30)                  #### Make up a random sequence #

         # Create a dict for this Quizly exercise
         question_answer_dict = {}
         question_answer_dict['unit_id'] = quizly_unit_id
         question_answer_dict['lesson_id'] = quizly_lesson_id
         question_answer_dict['sequence'] = quizly_sequence       # Not given
         question_answer_dict['question_id'] = instance_id        # Use instance_id as quid
         if instance_id in self.quizly_desc:
             question_answer_dict['description'] = self.quizly_desc[instance_id]
         else:
             question_answer_dict['description'] = 'Quizly ' + instance_id
         question_answer_dict['question_type'] = quizly_type
         question_answer_dict['timestamp'] = timestamp
         question_answer_dict['answers'] = [quizly_answer]   # T or F
         question_answer_dict['score'] = quizly_score
         question_answer_dict['weighted_score'] = quizly_score
         question_answer_dict['tallied'] = False
         question_answer_dict['choices'] = [{'text':'T','score':1},{'text':'F','score': 0}]   # Quizly's are T/F

         # Use the instance_id to count the number of attempts for each Quizly exercise by a given student
         if not student.email in self.num_attempts_dict:
             self.num_attempts_dict[student.email] = {}
         if not instance_id in self.num_attempts_dict[student.email]:
             self.num_attempts_dict[student.email][instance_id] = 1
         else:
{'fraction_non_alphanumeric': 0.10458516676232384, 'fraction_numerical': 0.006249601428480326, 'mean_word_length': 2.7989341085271318, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '349326', 'n_tokens_mistral': 4677, 'n_tokens_neox': 4440, 'n_words': 1032}