<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""A script that parses raw data output by jsPASAT (jsPsych), compiles each
participant's data and creates/updates a file in jsPASAT's ``data``
directory.
"""
import os
import glob
import json

import pandas as pd
import numpy as np
from scipy import stats


PROJECT_DIR = os.path.abspath(os.path.join(__file__, '..', '..'))
DATA_DIR = os.path.join(PROJECT_DIR, 'data')
ROUND_NDIGITS = 9


def get_csv_paths(basedir, exp_stage):
    """Take base data directory and experiment stage. Return list of file paths.
    """
    glob_path = os.path.join(basedir, exp_stage, '*.csv')
    return glob.glob(glob_path)


def get_csv_as_dataframe(path):
    """Take CSV path. Return pandas dataframe.
    """
    return pd.DataFrame.from_csv(path, index_col='trial_index_global')


def compile_practice_data(df):
    """Take pandas dataframe and compile key variables. Return dict.
    """
    compiled_data = {}

    # participant ID
    participant_id_col = df['participant_id'].values
    compiled_data['id'] = participant_id_col[0]

    # was the second practice block completed successfully?
    passed_practice = ('0-0.5-0' in df['internal_chunk_id'].values)
    compiled_data['passed_practice'] = passed_practice

    # time taken to complete practice
    time_practice_ms = int(df.ix[df.last_valid_index()]['time_elapsed'])
    compiled_data['time_practice_ms'] = time_practice_ms

    return compiled_data


def get_response_from_json(string, question_number=0):
    """Take JSON string representing a survey response and decode.
    Return target question answer string.
    """
    decoder = json.JSONDecoder()
    resp_json = decoder.decode(string)
    target_question = "Q{}".format(question_number)
    resp = resp_json[target_question] if target_question in resp_json else None
    return resp


def summarize_pasat_chunk(df):
    """Take pandas dataframe representing raw PASAT chunk data and summarize.
    Return dict.
    """
    summary = {}

    block_type_col = df['block_type'].dropna().values
    summary['block_type'] = block_type_col[0]

    # summarize performance
    raw_trials = df.loc[df['trial_type'] == 'multi-stim-multi-response']
    trials = list(raw_trials['correct'].values)
    trials.pop(0)  # remove fixation data
    accuracy = float(trials.count(True)) / len(trials)
    summary['accuracy'] = round(accuracy, ROUND_NDIGITS)

    # affective ratings
    ratings_json = df.ix[df.last_valid_index()]['responses']
    raw_effort_rating = get_response_from_json(ratings_json)
    summary['effort_rating'] = int(raw_effort_rating[0])
    raw_discomfort_rating = get_response_from_json(ratings_json, 1)
    summary['discomfort_rating'] = int(raw_discomfort_rating[0])

    return summary


def compile_experiment_data(df):
    """Take pandas dataframe and compile key variables. Return dict.
    """
    compiled_data = {}

    # condition
    condition_col = df['condition'].values
    compiled_data['condition'] = condition_col[0]

    # blocks and block order
    block_order_col = df['block_order'].values
    block_order = block_order_col[0]
    blocks = block_order.split(',')
    compiled_data['block_order'] = block_order
    compiled_data['num_blocks'] = len(blocks)

    # anticipated questions
    anticipated_questions_index = [
        ('anticipated_enjoyment', 1),
        ('anticipated_performance', 2),
        ('anticipated_effort', 3),
        ('anticipated_discomfort', 4),
        ('anticipated_fatigue', 5)
    ]
    for label, i in anticipated_questions_index:
        response = get_response_from_json(df.ix[i]['responses'])
        compiled_data[label] = int(response[0])

    # PASAT accuracy and affective reports
    hard_accuracy = None
    medium_accuracy = None
    easy_accuracy = None
    hard_effort = None
    medium_effort = None
    easy_effort = None
    hard_discomfort = None
    medium_discomfort = None
    easy_discomfort = None

    effort_ratings = []
    discomfort_ratings = []
    accuracies = []
    medium_effort_ratings = []
    medium_discomfort_ratings = []
    medium_accuracies = []

    # collect and organize experiment data from experimental blocks
    # note: PASAT chunks start at chunk_id 0-0.3-0
    for i, block in enumerate(blocks, start=3):
        block_chunk_id = '0-0.{}-0'.format(i)
        block = df.loc[df['internal_chunk_id'] == block_chunk_id]
        block_summary = summarize_pasat_chunk(block)

        effort_ratings.append(block_summary['effort_rating'])
        discomfort_ratings.append(block_summary['discomfort_rating'])
        accuracies.append(block_summary['accuracy'])

        if block_summary['block_type'] == 'medium':
            medium_accuracies.append(block_summary['accuracy'])
            medium_effort_ratings.append(block_summary['effort_rating'])
            medium_discomfort_ratings.append(block_summary['discomfort_rating'])
        elif block_summary['block_type'] == 'hard':
            hard_accuracy = block_summary['accuracy']
            hard_effort = block_summary['effort_rating']
            hard_discomfort = block_summary['discomfort_rating']
        elif block_summary['block_type'] == 'easy':
            easy_accuracy = block_summary['accuracy']
            easy_effort = block_summary['effort_rating']
            easy_discomfort = block_summary['discomfort_rating']

    # compute medium block averages
    medium_accuracy = np.mean(medium_accuracies)
    compiled_data['medium_accuracy'] = round(medium_accuracy, ROUND_NDIGITS)
    medium_effort = np.mean(medium_effort_ratings)
    compiled_data['medium_effort'] = round(medium_effort, ROUND_NDIGITS)
    medium_discomfort = np.mean(medium_discomfort_ratings)
    compiled_data['medium_discomfort'] = round(medium_discomfort, ROUND_NDIGITS)

    # compute regression variables for medium blocks
    medium_blocks_range = range(1, len(medium_accuracies) + 1)
    medium_block_measures = [
        ('medium_accuracy', medium_accuracies),
        ('medium_effort', medium_effort_ratings),
        ('medium_discomfort', medium_discomfort_ratings)
    ]
    for measure_name, measure_values in medium_block_measures:
        print measure_name
        print measure_values
        print medium_blocks_range

        # compute linear regression stats
        measure_regress = stats.linregress(medium_blocks_range, measure_values)

        slope_var = '{}_slope'.format(measure_name)
        compiled_data[slope_var] = round(measure_regress.slope, ROUND_NDIGITS)
        intercept_var = '{}_intercept'.format(measure_name)
        compiled_data[intercept_var] = round(
            measure_regress.intercept, ROUND_NDIGITS)

    # assign other variables
    compiled_data['hard_accuracy'] = hard_accuracy
    compiled_data['hard_effort'] = hard_effort
    compiled_data['hard_discomfort'] = hard_discomfort
    compiled_data['easy_accuracy'] = easy_accuracy
    compiled_data['easy_effort'] = easy_effort
    compiled_data['easy_discomfort'] = easy_discomfort

    compiled_data['start_effort'] = effort_ratings[0]
    compiled_data['peak_effort'] = max(effort_ratings)
    compiled_data['end_effort'] = effort_ratings[-1]
    avg_effort = np.mean(effort_ratings)
    compiled_data['avg_effort'] = round(avg_effort, ROUND_NDIGITS)

    compiled_data['start_discomfort'] = discomfort_ratings[0]
    compiled_data['peak_discomfort'] = max(discomfort_ratings)
    compiled_data['end_discomfort'] = discomfort_ratings[-1]
    avg_discomfort = np.mean(discomfort_ratings)
    compiled_data['avg_discomfort'] = round(avg_discomfort, ROUND_NDIGITS)

    average_accuracy = np.mean(accuracies)
    compiled_data['avg_accuracy'] = round(average_accuracy, ROUND_NDIGITS)<commit_msg>Remove comments and refactor regression calculations<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""A script that parses raw data output by jsPASAT (jsPsych), compiles each
participant's data and creates/updates a file in jsPASAT's ``data``
directory.
"""
import os
import glob
import json

import pandas as pd
import numpy as np
from scipy import stats


PROJECT_DIR = os.path.abspath(os.path.join(__file__, '..', '..'))
DATA_DIR = os.path.join(PROJECT_DIR, 'data')
ROUND_NDIGITS = 9


def get_csv_paths(basedir, exp_stage):
    """Take base data directory and experiment stage. Return list of file paths.
    """
    glob_path = os.path.join(basedir, exp_stage, '*.csv')
    return glob.glob(glob_path)


def get_csv_as_dataframe(path):
    """Take CSV path. Return pandas dataframe.
    """
    return pd.DataFrame.from_csv(path, index_col='trial_index_global')


def compile_practice_data(df):
    """Take pandas dataframe and compile key variables. Return dict.
    """
    compiled_data = {}

    # participant ID
    participant_id_col = df['participant_id'].values
    compiled_data['id'] = participant_id_col[0]

    # was the second practice block completed successfully?
    passed_practice = ('0-0.5-0' in df['internal_chunk_id'].values)
    compiled_data['passed_practice'] = passed_practice

    # time taken to complete practice
    time_practice_ms = int(df.ix[df.last_valid_index()]['time_elapsed'])
    compiled_data['time_practice_ms'] = time_practice_ms

    return compiled_data


def get_response_from_json(string, question_number=0):
    """Take JSON string representing a survey response and decode.
    Return target question answer string.
    """
    decoder = json.JSONDecoder()
    resp_json = decoder.decode(string)
    target_question = "Q{}".format(question_number)
    resp = resp_json[target_question] if target_question in resp_json else None
    return resp


def summarize_pasat_chunk(df):
    """Take pandas dataframe representing raw PASAT chunk data and summarize.
    Return dict.
    """
    summary = {}

    block_type_col = df['block_type'].dropna().values
    summary['block_type'] = block_type_col[0]

    # summarize performance
    raw_trials = df.loc[df['trial_type'] == 'multi-stim-multi-response']
    trials = list(raw_trials['correct'].values)
    trials.pop(0)  # remove fixation data
    accuracy = float(trials.count(True)) / len(trials)
    summary['accuracy'] = round(accuracy, ROUND_NDIGITS)

    # affective ratings
    ratings_json = df.ix[df.last_valid_index()]['responses']
    raw_effort_rating = get_response_from_json(ratings_json)
    summary['effort_rating'] = int(raw_effort_rating[0])
    raw_discomfort_rating = get_response_from_json(ratings_json, 1)
    summary['discomfort_rating'] = int(raw_discomfort_rating[0])

    return summary


def compile_experiment_data(df):
    """Take pandas dataframe and compile key variables. Return dict.
    """
    compiled_data = {}

    # condition
    condition_col = df['condition'].values
    compiled_data['condition'] = condition_col[0]

    # blocks and block order
    block_order_col = df['block_order'].values
    block_order = block_order_col[0]
    blocks = block_order.split(',')
    compiled_data['block_order'] = block_order
    compiled_data['num_blocks'] = len(blocks)

    # anticipated questions
    anticipated_questions_index = [
        ('anticipated_enjoyment', 1),
        ('anticipated_performance', 2),
        ('anticipated_effort', 3),
        ('anticipated_discomfort', 4),
        ('anticipated_fatigue', 5)
    ]
    for label, i in anticipated_questions_index:
        response = get_response_from_json(df.ix[i]['responses'])
        compiled_data[label] = int(response[0])

    # PASAT accuracy and affective reports
    hard_accuracy = None
    medium_accuracy = None
    easy_accuracy = None
    hard_effort = None
    medium_effort = None
    easy_effort = None
    hard_discomfort = None
    medium_discomfort = None
    easy_discomfort = None

    effort_ratings = []
    discomfort_ratings = []
    accuracies = []
    medium_effort_ratings = []
    medium_discomfort_ratings = []
    medium_accuracies = []

    # collect and organize experiment data from experimental blocks
    # note: PASAT chunks start at chunk_id 0-0.3-0
    for i, block in enumerate(blocks, start=3):
        block_chunk_id = '0-0.{}-0'.format(i)
        block = df.loc[df['internal_chunk_id'] == block_chunk_id]
        block_summary = summarize_pasat_chunk(block)

        effort_ratings.append(block_summary['effort_rating'])
        discomfort_ratings.append(block_summary['discomfort_rating'])
        accuracies.append(block_summary['accuracy'])

        if block_summary['block_type'] == 'medium':
            medium_accuracies.append(block_summary['accuracy'])
            medium_effort_ratings.append(block_summary['effort_rating'])
            medium_discomfort_ratings.append(block_summary['discomfort_rating'])
        elif block_summary['block_type'] == 'hard':
            hard_accuracy = block_summary['accuracy']
            hard_effort = block_summary['effort_rating']
            hard_discomfort = block_summary['discomfort_rating']
        elif block_summary['block_type'] == 'easy':
            easy_accuracy = block_summary['accuracy']
            easy_effort = block_summary['effort_rating']
            easy_discomfort = block_summary['discomfort_rating']

    # compute medium block averages
    medium_accuracy = np.mean(medium_accuracies)
    compiled_data['medium_accuracy'] = round(medium_accuracy, ROUND_NDIGITS)
    medium_effort = np.mean(medium_effort_ratings)
    compiled_data['medium_effort'] = round(medium_effort, ROUND_NDIGITS)
    medium_discomfort = np.mean(medium_discomfort_ratings)
    compiled_data['medium_discomfort'] = round(medium_discomfort, ROUND_NDIGITS)

    # compute regression variables for medium blocks
    medium_blocks_range = range(1, len(medium_accuracies) + 1)
    medium_block_measures = [
        ('medium_accuracy', medium_accuracies),
        ('medium_effort', medium_effort_ratings),
        ('medium_discomfort', medium_discomfort_ratings)
    ]
    for measure_name, measure_values in medium_block_measures:
        measure_regress = stats.linregress(medium_blocks_range, measure_values)
        compiled_data['{}_slope'.format(measure_name)] = round(
            measure_regress.slope, ROUND_NDIGITS)
        compiled_data['{}_intercept'.format(measure_name)] = round(
            measure_regress.intercept, ROUND_NDIGITS)

    # assign other variables
    compiled_data['hard_accuracy'] = hard_accuracy
    compiled_data['hard_effort'] = hard_effort
    compiled_data['hard_discomfort'] = hard_discomfort
    compiled_data['easy_accuracy'] = easy_accuracy
    compiled_data['easy_effort'] = easy_effort
    compiled_data['easy_discomfort'] = easy_discomfort

    compiled_data['start_effort'] = effort_ratings[0]
    compiled_data['peak_effort'] = max(effort_ratings)
    compiled_data['end_effort'] = effort_ratings[-1]
    avg_effort = np.mean(effort_ratings)
    compiled_data['avg_effort'] = round(avg_effort, ROUND_NDIGITS)

    compiled_data['start_discomfort'] = discomfort_ratings[0]
    compiled_data['peak_discomfort'] = max(discomfort_ratings)
    compiled_data['end_discomfort'] = discomfort_ratings[-1]
    avg_discomfort = np.mean(discomfort_ratings)
    compiled_data['avg_discomfort'] = round(avg_discomfort, ROUND_NDIGITS)

    average_accuracy = np.mean(accuracies)
    compiled_data['avg_accuracy'] = round(average_accuracy, ROUND_NDIGITS)
{'fraction_non_alphanumeric': 0.10498430141287284, 'fraction_numerical': 0.004578754578754579, 'mean_word_length': 4.109959893048129, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '320839', 'n_tokens_mistral': 4871, 'n_tokens_neox': 4682, 'n_words': 897}