<jupyter_start><jupyter_text>Social desirability experiment data analysis: subgroup analysis Note this is a long notebook with mainly table outputs...<jupyter_code>import numpy as np
import pandas as pd

import utility as util
df_no_miss = pd.read_csv("../output/df_no_miss.csv")
df_no_miss.shape
df_no_miss.columns<jupyter_output><empty_output><jupyter_text>Recode age_group and education and exclude records with any demographic missing<jupyter_code>age_g_map = {'19-25': '19-35',
             '26-35': '19-35',
             '36-45': '36-55', 
             '46-55': '36-55',
             '56-65': '56+',
             '66+': '56+'}

education_map ={'1. Less than high school diploma': '1. Less than college',
                '2. High school diploma': '1. Less than college',
                '3. Some college': '1. Less than college',
                '4. Bachelor degree': '2. College+',
                '5. Graduate degree': '2. College+'}
df_no_miss['age_group_r'] = df_no_miss['age_group'].map(age_g_map)
df_no_miss['education_r'] = df_no_miss['education'].map(education_map)
# exclude records with missing demographic info
df_sub = df_no_miss.dropna(subset=['gender', 'marital', 'age_group', 'education'])
df_sub.shape
demo_cols = ['gender', 'marital', 'age_group_r', 'education_r']<jupyter_output><empty_output><jupyter_text>Subset data<jupyter_code>woman_df = df_sub.loc[df_sub['gender'] == '1. Woman']
man_df = df_sub.loc[df_sub['gender'] == '2. Man']

married_df = df_sub.loc[df_sub['marital'] == '1. Married']
not_married_df = df_sub.loc[df_sub['marital'] == '2. Not married']

young_df = df_sub.loc[df_sub['age_group_r'] == '19-35']
middle_df = df_sub.loc[df_sub['age_group_r'] == '36-55']
old_df = df_sub.loc[df_sub['age_group_r'] == '56+']

lscollege_df = df_sub.loc[df_sub['education_r'] == '1. Less than college']
college_df = df_sub.loc[df_sub['education_r'] == '2. College+']<jupyter_output><empty_output><jupyter_text>Testing the effect of question wording<jupyter_code>recoded_cols = ['gone_to_friend_r', 'had_visitors_r', 'had_close_contact_r', 'gone_outside_r']
excuse_statement_order = ['no_excuse', 'with_excuse']<jupyter_output><empty_output><jupyter_text>gender differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', woman_df, excuse_statement_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', man_df, excuse_statement_order)<jupyter_output><empty_output><jupyter_text>marital status differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', married_df, excuse_statement_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', not_married_df, excuse_statement_order)<jupyter_output><empty_output><jupyter_text>age group differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', young_df, excuse_statement_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', middle_df, excuse_statement_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', old_df, excuse_statement_order)<jupyter_output><empty_output><jupyter_text>education differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', lscollege_df, excuse_statement_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'excuse_statement_condition', college_df, excuse_statement_order)<jupyter_output><empty_output><jupyter_text>Testing the effect of response set<jupyter_code>response_set_order = ['two_response', 'three_response']<jupyter_output><empty_output><jupyter_text>gender differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', woman_df, response_set_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', man_df, response_set_order)<jupyter_output><empty_output><jupyter_text>marital differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', married_df, response_set_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', not_married_df, response_set_order)<jupyter_output><empty_output><jupyter_text>age group differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', young_df, response_set_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', middle_df, response_set_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', old_df, response_set_order)<jupyter_output><empty_output><jupyter_text>education differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', lscollege_df, response_set_order)
for col in recoded_cols:
    util.crosstab_chisq(col, 'response_set_condition', college_df, response_set_order)<jupyter_output><empty_output><jupyter_text>Comparing control vs. C<jupyter_code>df_sub_controlC = df_sub[df_sub["condition"].isin(["Control", "C"])]
woman_df_cc = df_sub_controlC.loc[df_sub_controlC['gender'] == '1. Woman']
man_df_cc = df_sub_controlC.loc[df_sub_controlC['gender'] == '2. Man']

married_df_cc = df_sub_controlC.loc[df_sub_controlC['marital'] == '1. Married']
not_married_df_cc = df_sub_controlC.loc[df_sub_controlC['marital'] == '2. Not married']

young_df_cc = df_sub_controlC.loc[df_sub_controlC['age_group_r'] == '19-35']
middle_df_cc = df_sub_controlC.loc[df_sub_controlC['age_group_r'] == '36-55']
old_df_cc = df_sub_controlC.loc[df_sub_controlC['age_group_r'] == '56+']

lscollege_df_cc = df_sub_controlC.loc[df_sub_controlC['education_r'] == '1. Less than college']
college_df_cc = df_sub_controlC.loc[df_sub_controlC['education_r'] == '2. College+']
conditions_control_C = ['Control', 'C']<jupyter_output><empty_output><jupyter_text>gender differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', woman_df_cc, conditions_control_C)
for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', man_df_cc, conditions_control_C)<jupyter_output><empty_output><jupyter_text>marital differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', married_df_cc, conditions_control_C)
for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', not_married_df_cc, conditions_control_C)<jupyter_output><empty_output><jupyter_text>age group differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', young_df_cc, conditions_control_C)
for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', middle_df_cc, conditions_control_C)
for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', old_df_cc, conditions_control_C)<jupyter_output><empty_output><jupyter_text>education differences<jupyter_code>for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', lscollege_df_cc, conditions_control_C)
for col in recoded_cols:
    util.crosstab_chisq(col, 'condition', college_df_cc, conditions_control_C)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11874825370215143, 'fraction_numerical': 0.01173512154233026, 'mean_word_length': 7.719853836784409, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '274679', 'license': 'MIT', 'n_tokens_mistral': 2752, 'n_tokens_neox': 2640, 'n_words': 430, 'path': 'report/7.3_SD_experiment_subgroup.ipynb', 'repo_name': 'rosalynnyang/fb_uas_study', 'size': '7158'}