<jupyter_start><jupyter_text>Lecture 23   Percentiles<jupyter_code># Manually compute the 55th percentile.
x = make_array(43, 20, 51, 7, 28, 34)
# Step 1. Sort the data
np.sort(x)
# Step 2. Figure out where 55th percentile would be.
# OR: 1 Line of Code
percentile(55, x)
# If we tried to compute which element to take...
55 / 100 * 6<jupyter_output><empty_output><jupyter_text>Sample Median<jupyter_code>sf = Table.read_table('san_francisco_2015.csv')
sf
# Who is making the most money
sf.sort('Total Compensation', descending=True).show(5)
# Who is making the least money
sf.sort('Total Compensation', descending=False).show(5)
min_salary = 10 * 20 * 52
sf = sf.where('Total Compensation', are.above(min_salary))
pop_median = percentile(50, sf.column('Total Compensation'))
pop_median
our_sample = sf.sample(300, with_replacement=False)
our_sample.show(5)
percentile(50, our_sample.column('Total Compensation'))
sf_bins = np.arange(0, 700000, 25000)
sf.hist('Total Compensation', bins=sf_bins)
plots.title('Population Distribution');
our_sample.hist('Total Compensation', bins=sf_bins)
plots.title('Sample Distribution');<jupyter_output><empty_output><jupyter_text>Variability of the Estimate<jupyter_code>def generate_sample_median(samp_size):
    our_sample = sf.sample(samp_size, with_replacement=False)
    return percentile(50, our_sample.column('Total Compensation'))
sample_median = generate_sample_median(300)
sample_median
error = our_sample_median - pop_median
error<jupyter_output><empty_output><jupyter_text>Quantifying Uncertainty<jupyter_code>sample_medians = make_array()

for i in np.arange(1000):
    new_median = generate_sample_median(300)
    sample_medians = np.append(sample_medians, new_median)
med_bins = np.arange(90000, 125001, 2500)
Table().with_column(
    'Sample Medians', sample_medians
).hist(bins = med_bins)

plots.scatter(pop_median, 0, color="red");
err_bins = np.arange(-15000, 12501, 2500)
Table().with_column(
    'Errors', sample_medians - pop_median
).hist(bins = err_bins)

plots.scatter(0, 0, color="red");<jupyter_output><empty_output><jupyter_text>Bootstrap<jupyter_code># Take a bootstrap (re)sample of size 300, WITH replacement
boot_sample = our_sample.sample(300, with_replacement=True)
boot_sample.hist('Total Compensation', bins=sf_bins)
plots.title('Bootstrap sample');

print("Population Median =       ", pop_median)
print("Our Sample Median =       ", sample_median)
print("Bootstrap Sample Median = ", 
      percentile(50,boot_sample.column('Total Compensation')))
def one_bootstrap_median():
    single_sample = our_sample.sample()
    return percentile(50, single_sample.column('Total Compensation'))
bootstrap_medians = make_array()
for i in np.arange(1000):
    new_median = one_bootstrap_median()
    bootstrap_medians = np.append(bootstrap_medians, new_median)
Table().with_column(
    'Bootstrap Medians', bootstrap_medians
).hist('Bootstrap Medians', bins=med_bins)

plots.scatter(pop_median, 0, color="red");
plots.scatter(sample_median, 0, color="blue");<jupyter_output><empty_output><jupyter_text>Confidence Intervals<jupyter_code># Make an interval based on the middle 95% of bootstrap samples

left = percentile(2.5, bootstrap_medians)
right = percentile(97.5, bootstrap_medians)

Table().with_column(
    'Bootstrap Medians', bootstrap_medians
).hist('Bootstrap Medians', bins=med_bins)

plots.plot([left, right], [0,0], color="gold",lw=3, zorder=1);
plots.scatter(pop_median, 0, color="red", zorder=2);
plots.scatter(sample_median, 0, color="blue", zorder=2);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12685938815604828, 'fraction_numerical': 0.037889419028908224, 'mean_word_length': 7.155606407322654, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '46521', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1188, 'n_words': 258, 'path': 'lec/lec23.ipynb', 'repo_name': 'ds-modules/Deepnote-demo', 'size': '3563'}