<jupyter_start><jupyter_text>Imports<jupyter_code>from plot_utils import *
import seaborn as sns

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Load Survey Data (Pilot)<jupyter_code>df_pilot = pd.read_csv('data/pilot_data_cleaned.csv')
df_pilot.Start = pd.to_datetime(df_pilot.Start.values)
df_pilot.End = pd.to_datetime(df_pilot.End.values)
df_pilot['duration_in_minutes'] = round((df_pilot.End-df_pilot.Start).astype('timedelta64[s]')/60.,2)<jupyter_output><empty_output><jupyter_text>Pilot Participation Plot<jupyter_code>df2 = df_pilot.groupby(by=['ID','Type']).size().reset_index()[['ID','Type']]\
    .groupby(by='Type').size().reset_index().rename(columns={0:'total', 'Type':'treatment'})
df2.replace({"treatment":{'C':'Control', 'T':'Typographical', 'P':'Phonological'}}, inplace=True)

p = participant_count_plot(df2)
p.save('participant_count_pilot.html')<jupyter_output><empty_output><jupyter_text>Load Survey Day (Experiment)<jupyter_code>df = pd.read_csv('data/results_cleaned_04092021.csv')
df.drop(columns=['Unnamed: 0'], inplace=True)
df['Start Date'] = pd.to_datetime(df['Start Date'].values)<jupyter_output><empty_output><jupyter_text>Live Study Participation Plot<jupyter_code># live study participation bar chart
p = participant_count_plot_live(df)
p.save('participant_count_live.html')
#p<jupyter_output><empty_output><jupyter_text>Demographics Analysis<jupyter_code>rend = get_missing_demographics(df)

with open("missing_demographic_data.html","w") as f:
    f.write(rend.render()) # df is the styled dataframe
    f.close()
#rend
# generate plot for "good" year data
p = get_good_demographic_year(df)
p.save('demographic_year_goodonly.html')
#p
# generate gender plot
p = get_demographic_gender(df)
p.save('demographic_gender.html')
#p
# generate country plot
p = get_demographic_country(df)
p.save('demographic_country.html')
#p
p = get_demographic_state(df)
p.save('demographic_state.html')
#p
p = get_demographic_student_status(df)
p.savefig("demographic_student.png", dpi=300)
#p.show()<jupyter_output><empty_output><jupyter_text>Survey Response Data Analysis<jupyter_code># get descriptive statistics for main survey data (Amazon Mechanical Turk)
cols = ['PromptTime','QuestionTime','wpm','Interest','Effective','Intelligence','Writing','Meet']
rend = get_descriptive_statistics(df[(df['Start Date'] < '2021-04-05')], cols)

with open("descriptive_statistics_amazon.html","w") as f:
    f.write(rend.render()) # df is the styled dataframe
    f.close()

# get descriptive statistics for main survey data (Berkeley XLab)
rend = get_descriptive_statistics(df[(df['Start Date'] > '2021-04-05')], cols)

with open("descriptive_statistics_xlab.html","w") as f:
    f.write(rend.render()) # df is the styled dataframe
    f.close()
# generate Likert variance plot
p = get_likert_variance(df)
p.save('likert_variance.html')
#p
# generate likert counts by group
p = get_likert_counts_by_group(df)
p.save('likert_group_counts.html')
#p
# generate WPM plot by group
p = get_wpm_plot(df)
p.save('wpm_by_group.html')
#p<jupyter_output><empty_output><jupyter_text>Generate Divergence Plots (XLab Only)<jupyter_code>df2 = get_divergence_data(df[(df['Start Date'] > "2021-04-05")])

c1 = macro_diverge_plot(df2, "effective", "") #"Did the author effectively communicate their message?")
c2 = macro_diverge_plot(df2, "writing", "") # "Do you think this author has strong writing skills?") 
c3 = macro_diverge_plot(df2, "intelligence", "") #"What is your judgment of the author's level of intelligence?")

c1.save('divergence_effective.html')
c2.save('divergence_writing.html')
c3.save('divergence_intelligence.html')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14993197278911566, 'fraction_numerical': 0.014965986394557823, 'mean_word_length': 8.548051948051947, 'pattern_counts': {'":': 1, '<': 34, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '116635', 'license': 'MIT', 'n_tokens_mistral': 1405, 'n_tokens_neox': 1330, 'n_words': 228, 'path': 'visualizations/w241_final._eda.ipynb', 'repo_name': 'afogarty85/typos', 'size': '3675'}