<jupyter_start><jupyter_text>Libraries<jupyter_code>import pandas as pd
import numpy as np
import scipy.stats as stat

import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = (18.5 * 0.65, 10.5 * 0.65)

from mlgear.utils import show, display_columns
from surveyweights import run_weighting_iteration, run_weighting_scheme, normalize_weights, get_census<jupyter_output><empty_output><jupyter_text>Load Processed Data<jupyter_code>survey = pd.read_csv('responses_processed.csv')<jupyter_output><empty_output><jupyter_text>US National Weighting<jupyter_code>demographics = ['gender', 'race', 'education', 'urban_rural', 'income', 'age', 'vote2016', 'region',
                'gss_trust', 'gss_bible', 'gss_spanking']
run_weighting_iteration(survey, weigh_on=demographics, census='US')
output = run_weighting_scheme(survey, iters=50, weigh_on=demographics, census='US', verbose=1, early_terminate=False)
survey = output['final_df']
_ = run_weighting_iteration(survey, weigh_on=demographics, census='US')<jupyter_output>## gender ##
Female               1.002508
Male                 0.994053
Other                0.994926
Prefer not to say    1.008513
dtype: float64
ERROR: 0.0042184376889861566
-
-
## race ##
Asian or Asian American      1.009731
Black or African American    0.997755
Hispanic or Latino           1.003034
Other                        0.997009
White or Caucasian           0.992471
dtype: float64
ERROR: 0.006944254939915727
-
-
## education ##
Completed graduate school     0.999048
Graduated from college        0.997424
Graduated from high school    0.999930
Less than high school         1.005324
Some college, no degree       0.998274
dtype: float64
ERROR: 0.0017468454774673683
-
-
## urban_rural ##
Rural       0.992383
Suburban    0.999014
Urban       1.008603
dtype: float64
ERROR: 0.004306778714232753
-
-
## income ##
Between $100,000 and $150,000    1.002978
Between $15,000 and $49,999      0.992404
Between $50,000 and $74,999      0.998029
Between $75,000 and $99,999      0.997671
Ove[...]<jupyter_text>Likely Voter<jupyter_code>survey['lv_index'] = 0
survey['rv_index'] = 0
survey.loc[survey['lv_thought'] == 'Quite a lot', 'lv_index'] += 1
survey.loc[survey['lv_thought'] == 'Some', 'lv_index'] += 0.7
survey.loc[(survey['lv_plan_vote'] == 'Yes'), 'lv_index'] += 1
survey.loc[(survey['lv_likely'] == 'Very likely'), 'lv_index'] += 1
survey.loc[(survey['lv_likely'] == 'Likely'), 'lv_index'] += 0.7
survey.loc[(survey['lv_likely'] == 'Somewhat likely'), 'lv_index'] += 0.4
survey.loc[(survey['lv_likely'] == 'Neither likely nor unlikely'), 'lv_index'] += 0.2
survey.loc[(survey['lv_likely'] == 'Somewhat unlikely'), 'lv_index'] += 0.1
survey.loc[(survey['lv_likely'] == 'Unlikely'), 'lv_index'] += 0.05
survey.loc[(survey['age'] == '18-24'), 'lv_index'] += 1
survey.loc[(survey['age'] != '18-24') & (survey['vote2016'] != 'Did not vote'), 'lv_index'] += 1
survey.loc[survey['vote_method_plan'].isin(['In person on election day',
                                            'In person before election day',
                                            'By mail, and I’ve already requested and received my mail-in ballot']),
           'lv_index'] += 1
survey.loc[survey['vote_method_plan'] == 'By mail, and I’ve already requested my mail-in ballot', 'lv_index'] += 0.8
survey.loc[survey['vote_method_plan'] == 'By mail, but I haven\'t requested my mail-in ballot yet', 'lv_index'] += 0.4
survey.loc[survey['lv_already_voted'] == 'Yes', 'lv_index'] = survey['lv_index'].max()
survey.loc[~(survey['lv_registered'] == 'Yes'), 'lv_index'] = 0
survey.loc[(survey['lv_registered'] == 'Yes'), 'rv_index'] = 1

# https://www.pewresearch.org/methods/2016/01/07/measuring-the-likelihood-to-vote/
perry_gallup_loadings = {7: 0.83, 6: 0.63, 5: 0.59, 4: 0.4, 3: 0.34, 2: 0.23, 1: 0.13, 0: 0.11}
survey['lv_index'] = survey['lv_index'].apply(lambda l: perry_gallup_loadings[int(np.round(l * 2 * 0.7))])
plt.hist(survey['lv_index'], bins=7)
plt.show()
survey['lv_index'].value_counts()
survey['lv_weight'] = normalize_weights(survey['weight'] * survey['lv_index'])
survey['rv_weight'] = normalize_weights(survey['weight'] * survey['rv_index'])
print(survey['lv_weight'].mean())
print(survey['rv_weight'].mean())<jupyter_output>1.0
1.0<jupyter_text>Alternative Weighting<jupyter_code>census = get_census()
survey_ = survey.copy()
survey_['social_twitter'] = survey_['social_twitter'].astype(str)
census['social_twitter'] = {'False': 0.8, 'True': 0.22}
survey_['social_fb'] = survey_['social_fb'].astype(str)
census['social_fb'] = {'False': 0.31, 'True': 0.69}
census
demographics = ['gender', 'race', 'education', 'urban_rural', 'income', 'age', 'vote2016', 'region',
                'gss_trust', 'gss_bible', 'gss_spanking', 'social_twitter', 'social_fb']
run_weighting_iteration(survey_, weigh_on=demographics, census='US')
output = run_weighting_scheme(survey_, iters=50, weigh_on=demographics, census='US', verbose=1, early_terminate=False)
survey_ = output['final_df']
_ = run_weighting_iteration(survey_, weigh_on=demographics, census='US')
survey_['lv_weight'] = normalize_weights(survey_['weight'] * survey_['lv_index'])
survey_['lv_weight_alt'] = survey_['lv_weight']
survey_.loc[(~survey['voted2016']) & (survey_['vote_trump_biden'] == 'Donald Trump, the Republican'), 'lv_weight_alt'] *= 1.662
survey_['lv_weight_alt'] = normalize_weights(survey_['lv_weight_alt'])
survey['lv_weight_alt'] = survey_['lv_weight_alt']<jupyter_output><empty_output><jupyter_text>Trump 2020 Weighting<jupyter_code>vote2020 = {'vote_trump_biden': {'<NAME>, the Republican': 0.475, '<NAME>, the Democrat': 0.51}}
output = run_weighting_iteration(survey, census=vote2020, weigh_on=['vote_trump_biden'], verbose=0)
weights2020 = output['weights']['vote_trump_biden']
weights2020 = survey['vote_trump_biden'].astype(str).replace(weights2020)
survey['lv_weight_2020'] = survey['lv_weight'] * weights2020
survey['lv_weight_2020'] = normalize_weights(survey['lv_weight_2020'])<jupyter_output><empty_output><jupyter_text>Save<jupyter_code>survey.to_csv('responses_processed_national_weighted.csv', index=False)<jupyter_output><empty_output><jupyter_text>Analysis of Weights<jupyter_code>survey_ = survey.fillna('NA - Not presented')
for var in sorted(list(survey.columns)):
    print('## {} ##'.format(var))
    print('unweighted')
    print(survey_[var].value_counts(normalize=True)[sorted(survey_[var].unique())] * 100)
    print('-')
    print('weighted')
    print(survey_[var].value_counts(normalize=True) * survey_.groupby(var)['weight'].mean() * 100)
    print('-')
    print('weighted + RV')
    print(survey_[var].value_counts(normalize=True) * survey_.groupby(var)['rv_weight'].mean() * 100)
    print('-')
    print('weighted + LV')
    print(survey_[var].value_counts(normalize=True) * survey_.groupby(var)['lv_weight'].mean() * 100)
    print('-')
    print('weighted + LV alt')
    print(survey_[var].value_counts(normalize=True) * survey_.groupby(var)['lv_weight_alt'].mean() * 100)
    print('-')
    print('weighted + LV alt 2020')
    print(survey_[var].value_counts(normalize=True) * survey_.groupby(var)['lv_weight_2020'].mean() * 100)
    print('-')
    print('-')
    print('-')<jupyter_output>## age ##
unweighted
18-24    22.765052
25-44    59.436448
45-64    14.940199
65+       2.858301
Name: age, dtype: float64
-
weighted
18-24    13.035747
25-44    35.027048
45-64    34.800829
65+      17.136376
dtype: float64
-
weighted + RV
18-24    11.503112
25-44    35.313211
45-64    36.056452
65+      17.127225
dtype: float64
-
weighted + LV
18-24    11.734802
25-44    33.556019
45-64    36.301678
65+      18.407501
dtype: float64
-
weighted + LV alt
18-24    12.322422
25-44    33.799382
45-64    36.156934
65+      17.721261
dtype: float64
-
weighted + LV alt 2020
18-24    11.355815
25-44    33.415674
45-64    36.505785
65+      18.722726
dtype: float64
-
-
-
## agree_11hrs_sleep ##
unweighted
Agree          5.817961
Disagree      90.675046
Don't know     3.506994
Name: agree_11hrs_sleep, dtype: float64
-
weighted
Agree          7.604401
Disagree      89.495804
Don't know     2.899795
dtype: float64
-
weighted + RV
Agree          8.032859
Disagree      89.296496
Don't know     2.67[...]<jupyter_text>California National Weighting<jupyter_code>ca_national_survey = survey.copy()
demographics = ['gender', 'race', 'education', 'income', 'age', 'vote2016']
output = run_weighting_iteration(ca_national_survey, census='US_CA', weigh_on=['vote2016'])
ca_potus_weights = output['weights']['vote2016']
ca_potus_weights = ca_national_survey['vote2016'].astype(str).replace(ca_potus_weights)
ca_national_survey['weight'] = normalize_weights(ca_national_survey['weight'] * ca_potus_weights)
ca_national_survey['lv_weight'] = normalize_weights(ca_national_survey['weight'] * ca_national_survey['lv_index'])
ca_national_survey['rv_weight'] = normalize_weights(ca_national_survey['weight'] * ca_national_survey['rv_index'])
run_weighting_iteration(ca_national_survey, census='US_CA', weigh_on=demographics)
ca_national_survey['lv_weight_alt'] = ca_national_survey['lv_weight']
ca_national_survey.loc[(~ca_national_survey['voted2016']) & (ca_national_survey['vote_trump_biden'] == 'Donald Trump, the Republican'), 'lv_weight_alt'] *= 1.662
ca_national_survey['lv_weight_alt'] = normalize_weights(ca_national_survey['lv_weight_alt'])

vote2020 = {'vote_trump_biden': {'<NAME>, the Republican': 0.336, '<NAME>, the Democrat': 0.643}}
output = run_weighting_iteration(ca_national_survey, census=vote2020, weigh_on=['vote_trump_biden'], verbose=0)
weights2020 = output['weights']['vote_trump_biden']
weights2020 = ca_national_survey['vote_trump_biden'].astype(str).replace(weights2020)
ca_national_survey['lv_weight_2020'] = ca_national_survey['lv_weight'] * weights2020
ca_national_survey['lv_weight_2020'] = normalize_weights(ca_national_survey['lv_weight_2020'])
ca_national_survey.to_csv('responses_processed_ca_weighted.csv', index=False)<jupyter_output><empty_output><jupyter_text>California State Weighting<jupyter_code>ca_state_survey = survey.copy()
ca_state_survey = ca_state_survey.loc[ca_state_survey['state'] == 'California']
demographics = ['gender', 'race', 'education', 'income', 'age', 'vote2016']
run_weighting_iteration(ca_state_survey, weigh_on=demographics, census='US_CA')
output = run_weighting_scheme(ca_state_survey, iters=30, weigh_on=demographics, census='US_CA', verbose=1)
ca_state_survey = output['final_df']
ca_state_survey['rv_weight'] = normalize_weights(ca_state_survey['weight'] * ca_state_survey['rv_index'])
ca_state_survey['lv_weight'] = normalize_weights(ca_state_survey['weight'] * ca_state_survey['lv_index'])
_ = run_weighting_iteration(ca_state_survey, weigh_on=demographics, census='US_CA')
ca_state_survey['lv_weight_alt'] = ca_state_survey['lv_weight']
ca_state_survey.loc[(~ca_state_survey['voted2016']) & (ca_state_survey['vote_trump_biden'] == 'Donald Trump, the Republican'), 'lv_weight_alt'] *= 1.662
ca_state_survey['lv_weight_alt'] = normalize_weights(ca_state_survey['lv_weight_alt'])

vote2020 = {'vote_trump_biden': {'Donald Trump, the Republican': 0.336, '<NAME>, the Democrat': 0.643}}
output = run_weighting_iteration(ca_state_survey, census=vote2020, weigh_on=['vote_trump_biden'], verbose=0)
weights2020 = output['weights']['vote_trump_biden']
weights2020 = ca_state_survey['vote_trump_biden'].astype(str).replace(weights2020)
ca_state_survey['lv_weight_2020'] = ca_state_survey['lv_weight'] * weights2020
ca_state_survey['lv_weight_2020'] = normalize_weights(ca_state_survey['lv_weight_2020'])
ca_state_survey.to_csv('responses_processed_ca_state_ca_weighted.csv', index=False)<jupyter_output><empty_output><jupyter_text>Texas National Weighting<jupyter_code>tx_national_survey = survey.copy()
output = run_weighting_iteration(tx_national_survey, census='US_TX', weigh_on=['vote2016'])
tx_potus_weights = output['weights']['vote2016']
tx_potus_weights = tx_national_survey['vote2016'].astype(str).replace(tx_potus_weights)
tx_national_survey['weight'] = normalize_weights(tx_national_survey['weight'] * tx_potus_weights)
tx_national_survey['lv_weight'] = normalize_weights(tx_national_survey['weight'] * tx_national_survey['lv_index'])
tx_national_survey['rv_weight'] = normalize_weights(tx_national_survey['weight'] * tx_national_survey['rv_index'])
run_weighting_iteration(tx_national_survey, census='US_TX', weigh_on=demographics)
tx_national_survey['lv_weight_alt'] = tx_national_survey['lv_weight']
tx_national_survey.loc[(~tx_national_survey['voted2016']) & (tx_national_survey['vote_trump_biden'] == 'Donald Trump, the Republican'), 'lv_weight_alt'] *= 1.662
tx_national_survey['lv_weight_alt'] = normalize_weights(tx_national_survey['lv_weight_alt'])

vote2020 = {'vote_trump_biden': {'Donald Trump, the Republican': 0.521, '<NAME>, the Democrat': 0.464}}
output = run_weighting_iteration(tx_national_survey, census=vote2020, weigh_on=['vote_trump_biden'], verbose=0)
weights2020 = output['weights']['vote_trump_biden']
weights2020 = tx_national_survey['vote_trump_biden'].astype(str).replace(weights2020)
tx_national_survey['lv_weight_2020'] = tx_national_survey['lv_weight'] * weights2020
tx_national_survey['lv_weight_2020'] = normalize_weights(tx_national_survey['lv_weight_2020'])
tx_national_survey.to_csv('responses_processed_tx_weighted.csv', index=False)<jupyter_output><empty_output><jupyter_text>Texas State Weighting<jupyter_code>tx_state_survey = survey.copy()
tx_state_survey = tx_state_survey.loc[tx_state_survey['state'] == 'Texas']
demographics = ['vote2016', 'gender', 'race', 'education', 'income', 'age']
run_weighting_iteration(tx_state_survey, weigh_on=demographics, census='US_TX')
output = run_weighting_scheme(tx_state_survey, iters=50, weigh_on=demographics, census='US_TX', verbose=1)
tx_state_survey = output['final_df']
tx_state_survey['lv_weight'] = normalize_weights(tx_state_survey['weight'] * tx_state_survey['lv_index'])
tx_state_survey['rv_weight'] = normalize_weights(tx_state_survey['weight'] * tx_state_survey['rv_index'])
_ = run_weighting_iteration(tx_state_survey, weigh_on=demographics, census='US_TX')
tx_state_survey['vote2016'].value_counts(normalize=True) * 100
tx_state_survey['vote2016'].value_counts(normalize=True) * tx_state_survey.groupby('vote2016')['weight'].mean() * 100
tx_state_survey['lv_weight_alt'] = tx_state_survey['lv_weight']
tx_state_survey.loc[(~tx_state_survey['voted2016']) & (tx_state_survey['vote_trump_biden'] == 'Donald Trump, the Republican'), 'lv_weight_alt'] *= 1.662
tx_state_survey['lv_weight_alt'] = normalize_weights(tx_state_survey['lv_weight_alt'])

vote2020 = {'vote_trump_biden': {'Donald Trump, the Republican': 0.521, '<NAME>, the Democrat': 0.464}}
output = run_weighting_iteration(tx_state_survey, census=vote2020, weigh_on=['vote_trump_biden'], verbose=0)
weights2020 = output['weights']['vote_trump_biden']
weights2020 = tx_state_survey['vote_trump_biden'].astype(str).replace(weights2020)
tx_state_survey['lv_weight_2020'] = tx_state_survey['lv_weight'] * weights2020
tx_state_survey['lv_weight_2020'] = normalize_weights(tx_state_survey['lv_weight_2020'])
tx_state_survey['vote2016'].value_counts(normalize=True) * tx_state_survey.groupby('vote2016')['lv_weight_alt'].mean() * 100
tx_state_survey['vote2016'].value_counts(normalize=True) * tx_state_survey.groupby('vote2016')['lv_weight_2020'].mean() * 100
tx_state_survey.to_csv('responses_processed_tx_state_tx_weighted.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15339956134692298, 'fraction_numerical': 0.07347439040123856, 'mean_word_length': 7.2331386086032925, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '178749', 'license': 'MIT', 'n_tokens_mistral': 6512, 'n_tokens_neox': 5598, 'n_words': 856, 'path': '(B) Weighting.ipynb', 'repo_name': 'rethinkpriorities/rp_2020_benchmarking_poll', 'size': '15529'}