<jupyter_start><jupyter_text>Exploratory: from an aggregated 2pp polling estimate to a 2pp election result. Table of Contents1&nbsp;&nbsp;Python setup2&nbsp;&nbsp;Election 2pp results data3&nbsp;&nbsp;Historic polling data  Python setup<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from io import StringIO<jupyter_output><empty_output><jupyter_text>Election 2pp results data<jupyter_code># https://www.aec.gov.au/elections/federal_elections/tpp-results.htm
e_results = """
Election 	NSW 	VIC 	QLD 	WA  	SA  	TAS 	ACT 	NT  	AUST
10.12.49 	49.2 	50.3 	57.3 	53.0 	49.4 	53.5 	NA  	NA  	51.0
28.4.51 	49.5 	49.3 	56.5 	54.4 	48.2 	52.6 	NA  	NA  	50.7
29.5.54 	47.0 	49.3 	55.2 	51.9 	47.3 	50.0 	NA  	NA  	49.3
10.12.55 	50.5 	59.2 	56.1 	56.5 	51.0 	52.5 	NA  	NA  	54.2
22.11.58 	50.2 	57.7 	58.6 	59.2 	50.2 	50.2 	NA  	NA  	54.1
9.12.61 	45.2 	55.5 	49.3 	56.5 	45.8 	44.0 	NA  	NA  	49.5
30.11.63 	50.7 	57.5 	51.9 	56.2 	46.7 	45.5 	NA  	NA  	52.6
26.11.66 	56.1 	60.7 	55.7 	54.3 	57.6 	46.1 	44.2 	NA  	56.9
25.10.69 	48.4 	55.1 	50.1 	47.4 	45.8 	43.9 	28.4 	59.2 	49.8
2.12.72  	44.6 	49.6 	50.5 	51.7 	47.3 	39.5 	32.0 	54.5 	47.3
18.5.74  	45.1 	49.4 	54.6 	51.5 	47.5 	44.6 	40.3 	52.7 	48.3
13.12.75 	53.2 	56.2 	60.2 	58.8 	55.1 	55.9 	50.7 	54.9 	55.7
10.12.77 	52.4 	55.5 	58.0 	60.1 	51.3 	56.2 	45.7 	52.5 	54.6
18.10.80 	49.6 	49.3 	53.1 	53.3 	50.4 	52.9 	41.4 	51.2 	50.4
5.3.83  	45.91 	45.50 	50.54 	45.00 	47.70 	56.42 	34.51 	48.13 	46.77
1.12.84  	47.09 	46.91 	52.29 	49.14 	48.33 	53.37 	38.04 	51.39 	48.23
11.7.87  	49.71 	47.70 	50.68 	49.09 	49.76 	53.10 	36.75 	47.78 	49.17
24.3.90  	47.87 	52.54 	49.81 	52.87 	50.50 	52.10 	41.46 	44.98 	50.10
13.3.93  	45.62 	48.20 	51.57 	53.98 	52.67 	45.35 	38.81 	44.69 	48.56
2.3.96  	52.56 	49.70 	60.22 	56.00 	57.26 	48.42 	44.54 	50.37 	53.63
3.10.98  	48.46 	46.47 	53.05 	50.54 	53.11 	42.68 	37.56 	49.43 	49.02
10.11.01 	51.66 	47.86 	54.86 	51.62 	54.08 	42.27 	38.92 	47.51 	50.95
09.10.04 	51.93 	51.00 	57.09 	55.40 	54.36 	45.81 	38.46 	47.85 	52.74
24.11.07  	46.32 	45.73 	49.56 	53.26 	47.60 	43.79 	36.60 	44.59 	47.30
21.08.10 	51.16 	44.69 	55.14 	56.41 	46.82 	39.38 	38.33 	49.26 	49.88
7.9.13  	54.35 	49.80 	56.98 	58.28 	52.36 	48.77 	40.09 	50.35 	53.49
02.07.16 	50.53 	48.17 	54.10 	54.66 	47.73 	42.64 	38.87 	42.94 	50.36
18.05.19 	51.78 	46.86 	58.44 	55.55 	49.29 	44.04 	38.39 	45.80 	51.53
"""
# get the above text data into a pandas DataFrame
e_results_df = pd.read_csv(StringIO(e_results), sep='\s+',
                           parse_dates=['Election'], dayfirst=True)
# fix pandas desire to parse dates before 1970 as being in the future
e_results_df['Election'] = e_results_df['Election'].where(
    e_results_df['Election'].dt.year < 2020, 
    other=e_results_df['Election'] 
    - pd.Timedelta(days=100*365.25)).dt.date
    # Note, pd.Timedelta() does not have a years argument

e_results_df<jupyter_output><empty_output><jupyter_text>Historic polling data<jupyter_code># aggregated Australian polling data provided by @EthanOfHouseK on twitter
polling_data = pd.read_excel('../data/Australian Federal Polling Database.xlsx')
polling_data.columns
# Will use mid-point of collection period to locate a poll in the time series
polling_data['MeanDate'] = (
    polling_data['StartDate'] 
    + (polling_data['EndDate'] - polling_data['StartDate']) / 2
).dt.date
# Let's look at the 2pp population estimates in this data
# There seems to be alot of them ...
tpp = ['MeanDate', 'Pollster', 'Published2pp', 'RespAlloc2pp', 
       'Raw2pp', 'Est2pp', '2pp', 'RoundedEst2pp',]
polling_data[tpp].sample(20).sort_index()
polling_data[tpp].tail(20).sort_index()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13520749665327977, 'fraction_numerical': 0.2741633199464525, 'mean_word_length': 2.248695652173913, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '646547', 'license': 'MIT', 'n_tokens_mistral': 2511, 'n_tokens_neox': 2031, 'n_words': 440, 'path': 'notebooks/Exploratory poll 2pp to election 2pp result.ipynb', 'repo_name': 'bpalmer4/Australian-Federal-Election-2022', 'size': '3735'}