<jupyter_start><jupyter_text><jupyter_code>!pip install -q condacolab
import condacolab
condacolab.install()
# check that everything works
import condacolab
condacolab.check()
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import mean_absolute_error, mean_squared_error
# define experimental and calculated values
# draw from normal distribution
x_mu, x_sigma = 10.0, 1.5 
x_values = np.random.normal(x_mu, x_sigma, 1000)

y_mu, y_sigma = 11.0, 1.5 
y_values = np.random.normal(y_mu, y_sigma, 1000)
plt.hist(x_values, alpha=0.5)
plt.hist(y_values, alpha=0.5)
plt.show()
# bootstrap metric
def bootstrap_metric(fct, x_values, y_values):
    assert callable(fct) == True
    bootstrapped_metric = []
    # bootstrap metric to generate test distribution
    for _ in range(1000):
        indices = np.random.choice(range(0, len(x_values)), size=len(x_values), replace=True)
        x_selection = np.take(x_values, indices)
        y_selection = np.take(y_values, indices)
        r = fct(x_selection, y_selection)
        bootstrapped_metric.append(r)
    
    # define 90% CI
    alpha = 10.0
    lower_p = alpha / 2.0
    # get value at or near percentile (take a look at the definition of percentile if 
    # you have less than 100 values to make sure you understand what is happening)
    lower = np.percentile(bootstrapped_metric, lower_p)
    upper_p = (100 - alpha) + (alpha / 2.0)
    upper = np.percentile(bootstrapped_metric, upper_p)
    # calculate true mean
    mean = fct(x_values, y_values)

    return mean, lower, upper

# bootstrap MAE
mean, lower, upper = bootstrap_metric(mean_absolute_error, x_values, y_values)
print(f'MAE:  {round(mean, 2):.2f} [{round(lower,2):.2f}, {round(upper,2):.2f}]')

# bootstrap RMSE
def calc_rmse(x_values, y_values):
    from sklearn.metrics import mean_squared_error
    return np.sqrt(mean_squared_error(x_values, y_values))
mean, lower, upper = bootstrap_metric(calc_rmse, x_values, y_values)
print(f'RMSE:  {round(mean, 2):.2f} [{round(lower,2):.2f}, {round(upper,2):.2f}]')<jupyter_output>MAE:  1.84 [1.77, 1.92]
RMSE:  2.35 [2.26, 2.44]
{'fraction_non_alphanumeric': 0.1167296786389414, 'fraction_numerical': 0.034026465028355386, 'mean_word_length': 4.345959595959596, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '193087', 'license': 'MIT', 'n_tokens_mistral': 822, 'n_tokens_neox': 761, 'n_words': 192, 'path': 'Bootstrapping.ipynb', 'repo_name': 'wiederm/notebooks', 'size': '2116'}