<jupyter_start><jupyter_text>Effect of Polls on Donations<jupyter_code>import os
import sys
import json
import math
import datetime as dt
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LinearRegression
from nltk.metrics import *
from linearmodels import PanelOLS
from linearmodels import RandomEffects
import warnings

warnings.filterwarnings('ignore')
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Simple Linear RegressionAmount raised in one week -> polling percentage in morning consult weekly poll<jupyter_code>polls = pd.read_csv(os.path.join("..","data","polls","president_primary_polls.csv"))
results = polls.loc[polls["cycle"]==2020]
results = results.loc[results["stage"]=="primary"]
results = results.loc[results["party"]=="DEM"]
results = results.loc[results["pollster"]=="Morning Consult"]
results = results.groupby(by=["candidate_name","created_at"])["pct"].mean()
results = results.unstack(level="candidate_name")
results.head()
# find the path to each fec file, store paths in a nested dict
fec_2020_paths = {}
base_path = os.path.join("..","data","fec_new","2020")
for party_dir in os.listdir(base_path):
    if(party_dir[0]!="."):
        fec_2020_paths[party_dir] = {}
        for cand_dir in os.listdir(os.path.join(base_path,party_dir)):
            if(cand_dir[0]!="."):
                fec_2020_paths[party_dir][cand_dir] = {}
                for csv_path in os.listdir(os.path.join(base_path,party_dir,cand_dir)):
                    if(csv_path.find("schedule_a")>=0):
                        fec_2020_paths[party_dir][cand_dir]["donations"] = \
                            os.path.join(base_path,party_dir,cand_dir,csv_path)
                    elif(csv_path.find("schedule_b")>=0):
                        fec_2020_paths[party_dir][cand_dir]["spending"] = \
                            os.path.join(base_path,party_dir,cand_dir,csv_path)
print(json.dumps(fec_2020_paths, indent=4))
#tmp_df = pd.read_csv(fec_2020_paths["democrat"]["buttigieg"]["donations"])
#tmp_df.columns
#sorted(list(tmp_df["contribution_receipt_date"].unique()))<jupyter_output><empty_output><jupyter_text>Map candidates names from donations to pollsIf you adopt this for another dataset, you'll just need to adjust line 4.<jupyter_code>Candidates_Full_Name = ['<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
                       '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
                       '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
                       '<NAME>','<NAME>','<NAME>','<NAME>','<NAME>','<NAME>',
                       '<NAME>','<NAME>', '<NAME>', '<NAME>']
Candidates_Last_Name = ['delaney','yang','castro','harris','booker','gabbard','warren','klobuchar','sanders','inslee',
                       'hickenlooper','orourke','gillibrand','messam','swalwell','gravel','buttigieg','moulton','biden',
                       'bennet','bullock','blasio','trump','hickenlooper','williamson','de la fuente', 'ryan']
#Market = ['National','International','New York City','San Francisco','Maryland','Des Moines','Denver','Cedar Rapids',
#          'San Mateo California','Colorado Springs','Las Vegas','Phoenix','Pueblo','Reno','Waterloo','Philadelphia',
#          'Japan','Costa Mesa California','Columbia','Portsmouth','Baltimore','Charlotte','Cincinnati','Boston',
#          'Roanoke','Toledo','Daytona Beach','Washington DC','Cleveland','Arlington Virginia','Tampa','Miami','Orlando',
#          'Chicago','Greenville','Milwaukee','Belmont','Raleigh','Asheville','Maryland','Goldsboro','Ames','Shaker Heights',
#          'Lake Shore Maryland','Lynchburg','Spartangburg','Norfolk','St. Petersburg','Durham','Virginia Beach','Hampton',
#          'Newport KY','Greenville']
Market = ['National']
tv_api_base   = "https://api.gdeltproject.org/api/v2/tv/"
tv_api_params = {
    "mode":"timelinevol",
    "format":"csv",
    "STARTDATETIME":"20180101000000",
    "ENDDATETIME":"20190715000000"
}
tv_api_params_string = "&".join(["{:s}={:s}".format(key,val) for key,val in tv_api_params.items()])

data = pd.DataFrame()
for ii, (last_name, full_name) in enumerate(zip(Candidates_Last_Name, Candidates_Full_Name)):
    print("\r{:0.0f}% {:s}{:s}".format(100*ii/len(Candidates_Last_Name), full_name, " "*20), end="")
    for location in Market:
        api_call = "{:s}tv?query=%22{:s}%22%20market:%22{:s}%22&{:s}".format(
            tv_api_base,
            last_name,
            location,
            tv_api_params_string
        )
        try:
            temp_data = pd.read_csv(api_call)
            temp_data['Candidate'] = full_name
            temp_data['Market'] = location

            data = data.append(temp_data, ignore_index = True)
        except Exception as e:
            print(e, end="")
print("\r100% {:s}".format(" "*50))

# fix date column name
find_colname_str = "date"
for colname in data.columns:
    if(colname[:len(find_colname_str)].lower()==find_colname_str):
        found_colname_str = colname
data.rename({found_colname_str:find_colname_str.title()},axis="columns", inplace=True)
data["Date"] = pd.to_datetime(data["Date"], format='%Y-%m-%d')

# mung tv data
data = data[["Candidate","Date","Series","Value"]]
tv_mentions = data.groupby(["Candidate","Date"])["Value"].mean()
tv_mentions = tv_mentions.unstack(level="Candidate")

# save tv data
gdelt_dir = os.path.join("..","data","gdelt")
if(not os.path.isdir(gdelt_dir)):
    os.mkdir(gdelt_dir)
tv_mentions.to_csv(os.path.join(gdelt_dir,"tv_mentions.csv"))
"{:s}tv?query=%22{:s}%22%20market:%22{:s}%22&{:s}".format(
    tv_api_base,
    last_name,
    location,
    tv_api_params_string
)
name_mapping = {}
poll_names = results.columns
media_names = list(tv_mentions.columns)
for candid in fec_2020_paths["democrat"].keys():
    # poll data
    comparison_scores = [edit_distance(x[-len(candid):].lower(), candid.lower()) for x in poll_names]
    val, idx = min((val, idx) for (idx, val) in enumerate(comparison_scores))
    poll_name_map = poll_names[idx]
    # media data
    comparison_scores = [edit_distance(x[-len(candid):].lower(), candid.lower()) for x in media_names]
    val, idx = min((val, idx) for (idx, val) in enumerate(comparison_scores))
    media_name_map = media_names[idx]
    # output
    name_mapping[candid] = (poll_name_map, media_name_map)
    print("{:s} -> {:s} (score: {:d})".format(candid, str(name_mapping[candid]), val))
    
# special cases
name_mapping["ryan"] = ("<NAME>", "<NAME>")
for cc in sorted(name_mapping.keys()):
    nn = name_mapping[cc][1]
    print(str(tuple(nn.split()))+",")
def make_normalized_col(df, col_name):
    new_col_name = "{:s}_normalized".format(col_name)
    daily_sums = df.groupby("date")[col_name].sum()
    dates = df.index.get_level_values('date')
    df[new_col_name] = daily_sums.loc[dates].values
    df[new_col_name] = df[col_name]/df[new_col_name]

dataset = pd.DataFrame()
for candid in fec_2020_paths["democrat"].keys():
    if("donations" in fec_2020_paths["democrat"][candid].keys()):
        
        # process donations dataset
        df = pd.read_csv(fec_2020_paths["democrat"][candid]["donations"])
        df["contribution_receipt_date"] = pd.to_datetime(df["contribution_receipt_date"]).dt.date
        df = df.loc[df["entity_type"]=="IND"]
        df = df.loc[df["contribution_receipt_amount"]<=2800]
        # donation count
        df_count = df.groupby(by="contribution_receipt_date")["contribution_receipt_amount"].count()
        df_count.name = "individual_donations"
        # donation sum
        df_sum = df.groupby(by="contribution_receipt_date")["contribution_receipt_amount"].sum()
        df_sum.name = "individual_donation_amount"
        # donations under $200
        df = df.loc[df["contribution_receipt_amount"]<=200]
        df_small = df.groupby(by="contribution_receipt_date")["contribution_receipt_amount"].count()
        df_small.name = "small_donor_count"
        df = pd.concat([df_count, df_sum, df_small], axis=1)
        df["candidate"] = candid
        
        #  process polling dataset

        candid_polls = results[name_mapping[candid][0]]
        candid_polls.name = "polling_percentage"
        candid_polls.index = pd.to_datetime(candid_polls.index).date
        df = df.join(candid_polls, how="left")
        df["polling_percentage"] = df["polling_percentage"].interpolate(limit_direction='both').fillna(0.0)
        df = df.rename_axis("date").reset_index()
        
        # process media dataset
        df["date"] = pd.to_datetime(df["date"], format='%Y-%m-%d')
        candid_tv = tv_mentions[name_mapping[candid][1]]
        candid_tv.name = "tv_mentions"
        df = df.merge(candid_tv, how="left", left_on="date", right_index=True)
        
        # append to main df
        dataset = dataset.append(df)
        
        
# munge data
#dataset = dataset.reset_index()
#dataset = dataset.rename(columns={"index": "date"})
dataset = dataset.set_index(["candidate","date"])
dataset["individual_donations"] = dataset["individual_donations"].fillna(0.0)
make_normalized_col(dataset, "individual_donations")
dataset["individual_donation_amount"] = dataset["individual_donation_amount"].fillna(0.0)
make_normalized_col(dataset, "individual_donation_amount")
dataset["small_donor_count"] = dataset["small_donor_count"].fillna(0.0)
make_normalized_col(dataset, "small_donor_count")
dataset["tv_mentions"] = dataset["tv_mentions"].fillna(0.0)
make_normalized_col(dataset, "tv_mentions")

# write data
dataset.to_csv("../data/master_dataset.csv")
dataset.to_pickle("../data/master_dataset.pkl")

# check data
dataset.head(15)
pd.read_pickle("../master_dataset.pkl")
n_candid = len(dataset["candidate"].unique())
n_features = 4
fig, axes = plt.subplots(nrows=n_candid,ncols=n_features,figsize=(12*n_features,4*n_candid),facecolor="white")
#don_max = round(max(dataset["individual_donations"])/(10**6))*(10**6)
don_max = max(dataset["individual_donations"])
linear_bins = np.arange(0, don_max, don_max/20)
log_bins = np.arange(0, math.log(don_max), math.log(don_max)/30)
poll_max = 30
poll_bins = np.arange(0, poll_max, poll_max/30)
tv_max = 1.5
tv_bins = np.arange(0, tv_max, tv_max/30)

dataset_norm = dataset.copy()
for colname in ["individual_donations","polling_percentage","tv_mentions"]:
    colname_min_val = dataset_norm[colname].min()
    colname_max_val = dataset_norm[colname].max()
    dataset_norm[colname] =(dataset_norm[colname]-colname_min_val)/(colname_max_val-colname_min_val)

for ii, (candid, ax) in enumerate(zip(sorted(dataset["candidate"].unique()), axes)):
    '''
    plt.sca(ax[0])
    plt.hist(x=hist_data, bins=linear_bins)
    plt.title(candid.title() + " Donations")
    plt.ylabel("Frequency")
    ax[0].xaxis.set_major_formatter(ticker.StrMethodFormatter('${x:,.0f}'))
    '''
    # donations
    plt.sca(ax[0])
    hist_data = dataset[dataset["candidate"]==candid]["individual_donations"]
    hist_data = np.log(hist_data)
    plt.hist(x=hist_data, bins=log_bins)
    plt.title(candid.title() + " Count of Donations")
    plt.ylabel("Frequency")
    plt.ylim(0,30)
    ax[0].xaxis.set_major_formatter(ticker.FormatStrFormatter("10^%0.0f"))
    # polls
    poll_data = dataset[dataset["candidate"]==candid]["polling_percentage"]
    plt.sca(ax[1])
    hist_data = np.log(poll_data)
    plt.hist(x=poll_data, bins=poll_bins)
    plt.title(candid.title() + " Polling")
    plt.ylabel("Frequency")
    plt.ylim(0,60)
    ax[1].xaxis.set_major_formatter(ticker.FormatStrFormatter("%0.0f%%"))
    # tv mentions
    tv_data = dataset[dataset["candidate"]==candid]["tv_mentions"]
    plt.sca(ax[2])
    hist_data = tv_data
    plt.hist(x=tv_data, bins=tv_bins)
    plt.title(candid.title() + " TV Mentions")
    plt.ylabel("Frequency")
    plt.ylim(0,50)
    ax[1].xaxis.set_major_formatter(ticker.FormatStrFormatter("%0.1f%%"))
    # time series
    if(ii==0):
        show_legend=True
    else:
        show_legend=False
    plt.sca(ax[3])
    dataset_norm[dataset_norm["candidate"]==candid].plot(
        x="date",
        y=["individual_donations","polling_percentage","tv_mentions"],
        style='.-',
        legend=True,
        ax=ax[3]
    )
    plt.title(candid.title() + " Time Series")
    plt.xticks([], [])
    plt.xlabel("")
    plt.ylim(0,1)
print("{:s}: {:0.0f} donations".format("Overall Average", dataset["individual_donations"].mean()))
for candid in sorted(list(dataset["candidate"].unique())):
    print("\t{:s}: {:0.0f} donations".format(candid, dataset[dataset["candidate"]==candid]["individual_donations"].mean()))
X_train = np.array(dataset["polling_percentage"]).reshape(-1, 1)
y_train = dataset["individual_donations"]
X_test = np.array(range(0,36)).reshape(-1, 1)
linear_fit = LinearRegression().fit(X_train, y_train)
y_pred = linear_fit.predict(X_test)
print(linear_fit.score(X_train, y_train))
panel_data = dataset.set_index(["candidate","date"])
fe_model = PanelOLS(
    panel_data["individual_donations"],
    panel_data[["tv_mentions", "polling_percentage"]]
)
res = fe_model.fit()
res
fix, ax = plt.subplots(figsize=(12,8))
plot_scale = (10**3,"K","Thousands")
plt.scatter(X_train, y_train/plot_scale[0],  color='black', alpha=0.2)
plt.plot(X_test, y_pred/plot_scale[0], color='blue', linewidth=3)
plt.ylim(0,800)
ax.yaxis.set_major_formatter(ticker.FormatStrFormatter("$%0.0f{:s}".format(plot_scale[1])))
ax.xaxis.set_major_formatter(ticker.FormatStrFormatter("%0.0f%%"))
plt.ylabel("Donations")
plt.xlabel("Polling Percentage")
plt.title("Predicting Donations with Polling Percentage")
panel_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17294770669003798, 'fraction_numerical': 0.017017236342389715, 'mean_word_length': 4.748530646515533, 'pattern_counts': {'":': 5, '<': 44, '<?xml version=': 0, '>': 46, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '283182', 'license': 'MIT', 'n_tokens_mistral': 5077, 'n_tokens_neox': 4785, 'n_words': 673, 'path': 'src/polls_to_donations_model.ipynb', 'repo_name': 'berkeley-politics-capstone/politics-capstone', 'size': '13897'}