<jupyter_start><jupyter_text>Hypothesis Tests<jupyter_code>import numpy as np
import pandas as pd
import scipy.stats as ss
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
data = np.loadtxt("../data/loaded_500.txt")
plt.hist(data, ec='k', bins=6, rwidth=0.25);
num_6 = sum(data == 6)
total = data.size
print('Number of sixes:', num_6)
# probability of rolling 98 or more 6's with a fair dice
n = np.arange(total)
prob_n = ss.binom.pmf(n, total, 1/6)

plt.plot(n, prob_n, c='b', lw=3, label='prob num');
plt.axvline(total/6, ls='--', lw=2, c='c', label='mean num');
plt.axvline(num_6, ls='--', color='r', lw=2, label='obs num');
plt.ylabel('probability');
plt.xlabel(f'number of sixes rolled out of {total} rolls');
plt.legend();
d = ss.binom(total, 1/6)
plt.plot(n, d.sf(n), c='b', lw=2);
plt.axvline(num_6, ls='--', color='r');
sf = d.sf(num_6)
plt.axhline(sf, ls='--', c='r');
plt.xlabel('number of sixes');
plt.ylabel('survival function');
plt.ylim(0,1);
print(f'only {sf * 100:.1f}% of the time with a fair dice would you roll a six more than {num_6} times')
df = pd.read_csv("../data/Meteorite_Landings.csv")
df.head()
df.info()
df2 = df.dropna(subset=['mass', 'year'])
df2 = df2[df2['mass'] > 0]
df2.info()
plt.figure()
plt.gca().set_facecolor('k')
sns.scatterplot(x='reclong', y='reclat', data=df2, color='g');
sns.histplot(x=np.log(df2['mass']), bins=100);
counts = df2['year'].value_counts()
sns.scatterplot(x=counts.index, y=counts);
plt.xlim(1980, 2020);
ms = np.linspace(-5, 20, 100)
logmass = np.log(df2['mass'])
mean, std = logmass.mean(), np.std(logmass)
pdf_norm = ss.norm.pdf(ms, mean, std)

plt.hist(logmass, bins=100, density=True);
plt.plot(ms, pdf_norm, 'r-', lw=3);
log_params = ss.lognorm.fit(logmass)
pdf_lognorm = ss.lognorm.pdf(ms, *log_params)

skew_params = ss.skewnorm.fit(logmass)
pdf_skewnorm = ss.skewnorm.pdf(ms, *skew_params)

plt.figure()
ax = plt.gca()
ax.set_facecolor('k')
plt.hist(logmass, bins=100, density=True, color='g', alpha=0.5, label='data');
plt.plot(ms, pdf_norm, 'y--', lw=3, zorder=3, label='normal');
plt.plot(ms, pdf_lognorm, 'r--', lw=3, zorder=2, label='lognorm');
plt.plot(ms, pdf_skewnorm, 'c-', lw=5, zorder=1, label='skewnorm');
plt.legend();
mass_of_doom = np.log((4/3) * np.pi * 500**3 * 1600 * 1000) # spherical approximation
print(mass_of_doom)
ms2 = np.linspace(-5, 40, 200)
plt.plot(ms2, ss.lognorm.logsf(ms2, *log_params), color='b', lw=3);
plt.axvline(mass_of_doom, ls='--', c='r', lw=2);
plt.xlabel('log mass');
plt.ylabel('log probability');
prob_small_mass = ss.lognorm.cdf(mass_of_doom, *log_params)
print(prob_small_mass)
frac_sky_monitored = 0.2 # only see some events
num_years = 1000
num_events = num_years * counts.max() / frac_sky_monitored
print(num_events)
prob_large_mass = 1 - prob_small_mass
print(prob_large_mass)
prob_bad = num_events * prob_large_mass
print(f'Probability that a >1km asteroid hits Earth in the next {num_years} years is {prob_bad * 100:.2f}%')
# null hypothesis: 39% of votes went to A and 61% of votes went to B
# alternative hypothesis: >39% of votes went to A and <61% went to B
n_votes = 100
sigma = np.sqrt(0.39 * 0.61 / n_votes)
reported = 0.39
sample = 0.48

xs = np.linspace(0, 1, 500)
ps = ss.norm.pdf(xs, sample, sigma)
ps /= sum(ps)

plt.plot(xs, ps, lw=3, color='blue', label='sample probability');
plt.axvline(reported, ls='--', lw=2, c='r', label='reported proportion');
plt.fill_between(xs, ps, 0, alpha=0.2, where=xs>=reported, label='prob');
plt.legend(loc='best');
prob_more = ss.norm.sf(reported, sample, sigma)
print(f'There is a {100 * prob_more:.2f}% chance that candidate A got more votes')
z_score = (sample - reported) / sigma
z_score_needed_for_significance = ss.norm.ppf(0.95)
print(f'{z_score:.3f} is larger than {z_score_needed_for_significance:.3f} so we are significant')
print(f'have p-value {ss.norm.sf(sample, reported, sigma):.3f}')<jupyter_output>1.845 is larger than 1.645 so we are significant
have p-value 0.033<jupyter_text>Changing the number of votes will affect the resultsi.e. less votes makes our p-value grow and more votes decreases our p-value and makes our significance increase<jupyter_code># what is the chance that candidate A got a significantly different amount of votes than reported?
xs = np.linspace(0, 1, 500)
ps = ss.norm.pdf(xs, reported, sigma)

plt.plot(xs, ps, lw=3, c='b', label='proportion uncertain');
plt.axvline(sample, ls='--', c='r', lw=2, label='sample');
limits = ss.norm.ppf([0.025, 0.975], reported, sigma)
plt.fill_between(xs, ps, 0, alpha=0.2, where=(xs<limits[0])|(xs>limits[1]), label='significant');
plt.legend();
plt.xlim(0.2, 0.65);
z_score = (sample - reported) / sigma
z_score_needed_for_significance = ss.norm.ppf(0.975)
print(f'{z_score:.3f} is less than {z_score_needed_for_significance:.3f} so we are not significant')

p_value_two_tailed = 2 * ss.norm.sf(z_score)
print(f'have p-value {p_value_two_tailed:.3f} which is > 0.05')
# chi square test
data = np.loadtxt("../data/loaded_500.txt")
unique, counts = np.unique(data, return_counts=True)
expected = data.size / 6
chi2_val = np.sum((counts - expected)**2 / expected)

chi2s = np.linspace(0, 15, 500)
prob = ss.chi2.pdf(chi2s, 5)

plt.plot(chi2s, prob, lw=3, c='b', label='Distribution');
plt.axvline(chi2_val, label='$\chi2$', ls='--', lw=2, c='r');
plt.fill_between(chi2s, prob, 0, where=(chi2s>=chi2_val), alpha=0.2);
plt.legend();
print(f'P-value is {ss.chi2.sf(chi2_val, 5):.3f}')
chisq, p = ss.chisquare(counts, expected)
print(f'we have a chi2 of {chisq:.2f} with a p-value of {p:.3f}')
rolls = np.loadtxt("../data/loaded_many_100.txt")
unique, counts = np.unique(rolls, return_counts=True)
counts = counts / counts.sum()
fair_rolls = np.random.randint(low=1, high=7, size=(500000, 10)).sum(axis=1)
unique2, count_fair = np.unique(fair_rolls, return_counts=True)
count_fair = count_fair / count_fair.sum()

plt.plot(unique, counts, lw=3, c='b', label='data');
plt.plot(unique2, count_fair, lw=3, c='r', label='fair');
plt.legend();
cdf_obs = counts.cumsum()
cdf_fair = count_fair.cumsum()

plt.plot(unique, cdf_obs, lw=2, c='b', label='Data');
plt.plot(unique2, cdf_fair, lw=2, c='r', ls='--', label='fair');
plt.legend();
statistic, pval = ss.ks_2samp(rolls, fair_rolls)
print(f'KS test statistic of {statistic:.3f}, p-value of {pval:.3f}')
statistic, critical_vals, sig_level = ss.anderson_ksamp([rolls, fair_rolls])
print(f'AD test statistic of {statistic:.3f}, sig-level of {sig_level:.9f}')
print(f'Test critical values are {critical_vals}')
print('Sig-level analogous to p-value, and critical values are [25%, 10%, 5%, 2.5%, 1%]')<jupyter_output>AD test statistic of 5.335, sig-level of 0.002635535
Test critical values are [0.325 1.226 1.961 2.718 3.752 4.592 6.546]
Sig-level analogous to p-value, and critical values are [25%, 10%, 5%, 2.5%, 1%]
{'fraction_non_alphanumeric': 0.18482718219097832, 'fraction_numerical': 0.04686584651435267, 'mean_word_length': 5.617248062015504, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '531548', 'license': 'MIT', 'n_tokens_mistral': 3058, 'n_tokens_neox': 2762, 'n_words': 666, 'path': 'notebooks/hypothesis_testing.ipynb', 'repo_name': 'Joshua-Robison/StatisticalAnalysis', 'size': '6828'}