<jupyter_start><jupyter_text>Fetch data from serverRun `bin/fetch_data.py `, where VERSION can be found in config.txt.Data will be saved in `human_raw`, and then interpretted in `human`.<jupyter_code># N.B. all hacky and done quickly
version = '1.3'
load_dir = '../data/human/{}'.format(version)
save_dir = load_dir
print(os.listdir(load_dir))

responses = pd.read_csv("{}/html-button-response.csv".format(load_dir), index_col=0, squeeze=True)
surveys = pd.read_csv("{}/survey-multi-select.csv".format(load_dir), index_col=0, squeeze=True)
wids = set(responses.wid.tolist())

responses
stimuli = list(set(responses[responses['task'] == 'test_respond'].stimulus.to_list()))
stimuliParsed = []
for s in stimuli:
    temp_stim_list = s.split('images/')
    image_1 = temp_stim_list[1][:3]
    image_2 = temp_stim_list[2][:3]
    stimuliParsed.append((s, image_1, image_2))
# print(stimuliParsed, len(stimuliParsed))

raster_dict = {}
for s, i1, i2 in stimuliParsed:
    countArray = np.arange(9)
    t = responses[responses["stimulus"] == s]["response"].value_counts(sort=True)
#     print(t)
    for num, count in t.iteritems():
#         print(num, count)
        countArray[int(num)] = count
#     print(countArray)
#     print(t)
    raster_dict['{}_{}'.format(i1, i2)] = countArray
# print(raster_dict)    

square_len = int(np.ceil(np.sqrt(len(stimuliParsed))))
fig, ax = plt.subplots(square_len, square_len, figsize=(square_len*4, square_len*2),
                      sharex=True, sharey=True)

for i, (_, i1, i2) in enumerate(stimuliParsed):
#     print(i)
    x, y = np.unravel_index(i, (square_len, square_len))
#     print(x, y)
    ax[x, y].bar(np.arange(9), raster_dict['{}_{}'.format(i1, i2)])
    ax[x, y].set_title('{}_{}'.format(i1, i2))
    if y % square_len == 0:
#         print(x)
        ax[x, y].set_ylabel('Counts')
    if x == square_len-1:
#         print(y)
        ax[x, y].set_xlabel('Choices')
        try:
            ax[x, y].set_x_ticklabels(np.arange(1, 10))
        except:
            continue
plt.subplots_adjust(hspace=1.0)
def preprocess_trial_data(df):
    """Just print out stimuli, RT, decision for each participant."""
    df2 = df[df['task'] == 'test_respond']
    df2 = df2.reset_index()
    df2 = df2[['wid', 'rt','stimulus', 'response', 'question_order']]
    for i, r in df2.iterrows():
        temp_stim = r[2]
        temp_stim_list = temp_stim.split('images/')
        image_1 = temp_stim_list[1][:3]
        image_2 = temp_stim_list[2][:3]
        df2.iloc[i, 2] = '{} vs {}'.format(image_1, image_2)
        df2.iloc[i, -1] = temp_stim_list[1][2]
    df2 = df2.rename({'question_order': 'stimulus_set'}, axis='columns')
    return df2

def preprocess_participant_data(df, total_responses = 48):
    wids = list(set(df['wid']))
    data = []
    for wid in wids:
        dfTemp = df[df['wid']==wid]
        tempResponses = np.bincount(np.int_(dfTemp['response'][~np.isnan(dfTemp['response'])]),
                                   minlength=9)
        exclude_counts = (tempResponses > np.floor(0.9 * total_responses)).any()
        timeResponses = dfTemp['time_elapsed'][~np.isnan(dfTemp['response'])]
        time = max(timeResponses)/60000
        exclude_time =  time > 20
        tempSeries = pd.Series(data = {'wid': wid, 'counts': tempResponses, 
                                       'exclude_time': exclude_time,
                                       'exclude counts': exclude_counts,
                                      'exclude_time': exclude_time})
        data.append([wid, tempResponses, time,  exclude_counts, exclude_time])
    dfNew = pd.DataFrame(data, columns = ['wid', 'counts', 'time', 'exclude counts', 'exclude time'])
    return dfNew
        
cleanResponses = preprocess_trial_data(responses)
cleanResponses.to_csv('{}/html-button-response_processed_{}.csv'.format(save_dir, version))

excludeData = preprocess_participant_data(responses)
excludeData.to_csv('{}/exclusion_data_{}.csv'.format(save_dir, version))
print(cleanResponses)
exclude = []

for i, r in excludeData.iterrows():
    if r['exclude counts'] or r['exclude time']:
        print('excluding {}'.format(r['wid']))
        exclude.append(r['wid'])

print('exluding following participants: {}'.format(exclude))
print('average time was {}; range was {} to {}'.format(excludeData["time"].mean(), 
                                                      excludeData["time"].min(),
                                                      excludeData["time"].max()))
excludeData
def process_participant_data(df, wid):
    results = {}
    stimuli = ["A", "B", "C", "D"]
    
    for stim in stimuli:
        results[stim] = []
        
    subDf = df.loc[df['wid'] == wid]
    
    for index, row in subDf.iterrows():
#         print(row['stimulus'])
        newItems = row['stimulus'].split('_')
        stimulus_1 = newItems[0]
        stimulus_2 = newItems[1].split('vs ')[1]
#         print(stimulus_1, stimulus_2)
        if stimulus_1 == 'T':
#             print('choosing second stimulus')
            stimulus = stimulus_2
        elif stimulus_2 == 'T':
#             print('choosing first response')
            stimulus = stimulus_1
        results[stimulus].append(row['response'])
        
    for stim in stimuli:
        results[stim] = np.vstack(results[stim])
        
    results["C-A"] =  results["C"] -  results["A"]
    results["D-B"] =  results["D"] -  results["B"]
    results["(D-B)-(C-A)"] = results["D-B"] - results["C-A"]
    results["Max"] = (results["D-B"] > 0) & (results["C-A"] < 0)
    results["Min"] = (results["D-B"] < 0) & (results["C-A"] > 0)
    
#     print('results before')
#     print(results)
    for key in results.keys():
        if key in ["Max", "Min"]: # this is how the authors define it in the paper: only if effect on both trials
            results[key] = np.all(results[key], axis=0)[0]
        else:
            results[key] = np.mean(results[key], axis =0)[0]
            
#     print('results after')
#     print(results)
    return results

def process_participant_survey(df, wid):
    results = {}
    answers = ['Age', 'Gender', 'Math']
    subDf = df.loc[df['wid'] == wid]
    aR = subDf["response"].tolist()[0]
    for ans in answers:
        aR2 = aR.split("'{}':".format(ans))[1]
        endpoint = aR2.find(",")
        results[ans] = aR2[3:endpoint-2]
    try:
        results["Math Years"] = np.int(results["Math"].split('+')[0].split('-')[0])
    except:
        results["Math Years"] = 'Prefer not to say'
    return results

def process_data_set(df, wids, stimulus_set=False):
    setData = {}
    for wid in wids:
        if stimulus_set:
            df = df[df['stimulus_set'] == '{}'.format(stimulus_set)]
        setData[wid] = process_participant_data(df, wid)
    df = pd.DataFrame.from_dict(setData, orient='index')
    return df

def summarize_set_data(df):
    expResults = df[['A', 'B', 'C', 'D', 'C-A', 'D-B', '(D-B)-(C-A)']].mean()
    expResults["N_max"] = df["Max"].sum()
    expResults["N_min"] = df["Min"].sum()
    expResults["N"] = len(df)
    expResults["N_ind"] = expResults["N"] - expResults["N_max"] - expResults["N_min"]
    expResults["Var_set"] = np.var(df['(D-B)-(C-A)'])
    expResults["SD_set"] = np.sqrt(expResults["Var_set"])
    expResults["SEM_set"] = expResults["SD_set"] / np.sqrt(expResults["N"])
    expResults["Var_C-A_set"] = np.var(df['C-A'])
    expResults["Var_D-B_set"] = np.var(df['D-B'])
    return expResults

def summarize_all_sets(dataDF, wids, num_sets):
    dataSetList = []
    for s in np.arange(1, num_sets+1):
#         print(s)
        dataSetList.append(summarize_set_data(process_data_set(dataDF, wids, stimulus_set=s)))
#         print(dataSetList[-1]) 
    
    bigDF = pd.concat(dataSetList, axis=1).transpose()
    return bigDF


def compress_all_sets(setDF, otherDF):
    """Returns series object. setDF is set-level means and maxes. otherDF is 
    for variance calculation, and contains all answers. Both dfs should have had
    wids removed."""
    expResults = setDF[['A', 'B', 'C', 'D',  'C-A', 'D-B', '(D-B)-(C-A)']].mean()
    expResults["N_max"] = setDF["N_max"].sum()
    expResults["N_min"] = setDF["N_min"].sum()
    expResults["N"] = setDF["N"].sum()
    expResults["N_ind"] = expResults["N"] - expResults["N_max"] - expResults["N_min"]
    expResults["Var_all"] = np.var(otherDF['(D-B)-(C-A)'])
    expResults["SD_all"] = np.sqrt(expResults["Var_all"])
    expResults["Var_C-A_all"] = np.var(otherDF['C-A'])
    expResults["Var_D-B_all"] = np.var(otherDF['D-B'])
    expResults["SEM_all"] = expResults["SD_all"] / np.sqrt(expResults["N"])
    return expResults

study = 160
try:
    validWids = list(wids - set(exclude))
except Exception as e:
    print(e)
    validWids = wids
    
if len(validWids) > study:
    validWids.sort()
    to_drop = len(validWids) - study 
    print(" we have {} wids and study size is {}. \n dropping {}".format(len(validWids), study, to_drop))
    
    for i in np.arange(to_drop):
        # sorting should always make this consistent
        validWids.pop()

allDF = process_data_set(cleanResponses, validWids)

fig, ax = plt.subplots()
ax.hist(allDF['(D-B)-(C-A)'], color='k', bins=10)
ax.set_ylabel('Number of responses')
ax.set_xlabel('(D-B)-(C-A)')
plt.savefig('{}/response_distribution.png'.format(save_dir), bbox_inches="tight")
setDF = summarize_all_sets(cleanResponses, validWids, 6)
overallDF = compress_all_sets(setDF, allDF)
print(setDF)
print()
print(overallDF)
# print(allDF['(D-B)-(C-A)'])

setDF.to_pickle('{}/processed_data_set_level_{}.p'.format(save_dir, version))
overallDF.to_pickle('{}/processed_data_exp_level_{}.p'.format(save_dir, version))
num_sets=6
dataSetList = []
for s in np.arange(1, num_sets+1):
    dataSetList.append(process_data_set(cleanResponses, validWids, stimulus_set=s))
bigDFMAX = pd.concat(dataSetList, axis=1).iloc[:, [7, 16, 25, 34, 43, 52]].sum(axis=1)
#display(bigDFMAX[bigDFMAX + bigDFMIN > 0])
bigDFMIN = pd.concat(dataSetList, axis=1).iloc[:, [8, 17, 26, 35, 44, 53]].sum(axis=1)
#display(bigDFMIN[bigDFMAX + bigDFMIN > 0])
responders = bigDFMAX + bigDFMIN > 0
print(responders.sum())
display(bigDFMAX[responders] - bigDFMIN[responders])
finalAnalysis = bigDFMAX[responders] - bigDFMIN[responders]
print((finalAnalysis >= 0).sum())
print(np.mean(np.sign(finalAnalysis)))
print(np.sqrt(np.var(np.sign(finalAnalysis))))<jupyter_output>24<jupyter_text>"For each picture set, each subject was labeledas a MAX subject if he/she consistently (i.e., on both trials) rated A asmore similar to T than C, and D as more similar to T than B. They werelabeled MIN if the opposite pattern applied. If the subject’s two ratings ofa pair did not depict the same ordinal pattern with respect to its comparisonrating, then the response was not counted as either MAX or MIN.Because some subjects were therefore left unlabeled, the number of MAXsubjects plus the number of MIN subjects in Table 1 is less than the totalnumber of subjects."<jupyter_code># originalResults = pd.Series([5.8, 3.4, 5.2, 3.6, 0.8, 10, 5, 29], index = ['A', 'B', 'C', 'D', '(D-B)-(C-A)', 
#                                                                       "N_max", "N_min", "N"])
# combinedResults = pd.concat([originalResults, expResults], axis=1, keys=['Original', 'Ours'])
# combinedResults.to_pickle('{}/processed_data_summary_{}.p'.format(save_dir, version))
matplotlib.rcParams.update({'font.size': 18})
fig, ax = plt.subplots()
ax = dataDF.plot.scatter('D-B', 'C-A', c='Math Years', figsize=(10, 10), ax=ax)
ax.set_aspect('equal')

lims = [
    np.min([ax.get_xlim(), ax.get_ylim()]),  # min of both axes
    np.max([ax.get_xlim(), ax.get_ylim()]),  # max of both axes
]

# now plot both limits against eachother
ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)
ax.set_aspect('equal')
ax.set_xlim(lims)
ax.set_ylim(lims)
plt.savefig('plot_results_{}.png'.format(version))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17232704402515722, 'fraction_numerical': 0.014171907756813417, 'mean_word_length': 3.410502958579882, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '546425', 'license': 'MIT', 'n_tokens_mistral': 4396, 'n_tokens_neox': 4221, 'n_words': 860, 'path': 'analysis/interpret_data_processed.ipynb', 'repo_name': 'battleday/psirokuturk', 'size': '11925'}