<jupyter_start><jupyter_text>Notebook settings<jupyter_code>from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))

pd.set_option("precision", 3)<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code>df = pd.read_csv("data/calibration-2021-03-13.csv")
# Removing irrelevant columns, id, state and question
df = df[["user", "lo", "hi", "ci", "solution"]]
# z: How many std deviations away is the solution from the hi/lo inferred normal
df["z"] = np.divide(2*df["solution"]-(df["hi"]+df["lo"]), (df["hi"]-df["lo"])/norm.ppf(.95))
# The percentaliz based on the std_deviatins
df["z-percentile"] = df["z"].apply(norm.cdf)
# log(z): How many std deviations away is the solution from the hi/lo inferred lognormal
df["loglo"] = np.log(df["lo"])
df["loghi"] = np.log(df["hi"])
df["logsolution"] = np.log(df["solution"])
df["logz"] = np.divide(2*df["solution"]-(df["loghi"]+df["loglo"]), (df["loghi"]-df["loglo"])/norm.ppf(.95))
# log(z): How many std deviations away is the solution from the hi/lo inferred lognormal
df["logz-percentile"] = df["logz"].apply(norm.cdf)
df.head()
df.logz.isna().value_counts()<jupyter_output><empty_output><jupyter_text>z-score density<jupyter_code>norm.cdf(-1)
norm.ppf(0.05)
df[df["z-percentile"].between(0.05,.15)]
df.hist("z-percentile", bins=100, figsize=(40,5))
df.hist("logz-percentile", bins=100, figsize=(40,5))<jupyter_output><empty_output><jupyter_text>Auxiliary functions<jupyter_code>def get_user_table(user):
    return df[df["user"] == user][[col for col in df.columns if col != "user"]]<jupyter_output><empty_output><jupyter_text>Add metrics<jupyter_code>n_buckets = 20
buckets = np.divide(range(1,n_buckets,1),n_buckets)
print(buckets)
for ci_size in tqdm(buckets):
    alpha = np.divide(1-ci_size,2)
    df[f"ci-{ci_size}"] = df["z"].map(lambda x: x>norm.ppf(alpha) and x<norm.ppf(1-alpha))
    
for ci_size in tqdm(buckets):
    alpha = np.divide(1-ci_size,2)
    df[f"log_ci-{ci_size}"] = df["logz"].map(lambda x: x>norm.ppf(alpha) and x<norm.ppf(1-alpha))
df.head()<jupyter_output><empty_output><jupyter_text>Calculating the buckets<jupyter_code>def get_percentage(user, column):
    results = df[df["user"]==user][column].value_counts()
    return np.divide(results[True],results[True]+results[False])
users = df.user.unique()
df_metrics = pd.DataFrame(index = users)
for ci in buckets:
    df_metrics[ci] = df_metrics.index.map(lambda user: get_percentage(user,f"ci-{ci}"))
    
df_metrics.loc["perfect"] = buckets
df_metrics.T.style.background_gradient(cmap='viridis', axis=1)
df_metrics.loc["user3"].plot(legend=True, figsize=(20,5))
df_metrics.loc["perfect"].plot(legend=True, figsize=(20,5))
for user in df_metrics.index:
    df_metrics.loc[user].plot(legend=True, figsize=(20,5))
df_metrics["avg_abs_error"] = np.mean(np.abs(df_metrics[buckets] - buckets),axis=1)
df_metrics["bias"] = np.mean(df_metrics[buckets] - buckets,axis=1)
df_metrics.T.style.background_gradient(cmap='viridis', axis=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19200524246395806, 'fraction_numerical': 0.020642201834862386, 'mean_word_length': 7.410468319559229, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '608527', 'license': 'MIT', 'n_tokens_mistral': 1257, 'n_tokens_neox': 1189, 'n_words': 187, 'path': 'Calibration/Playground.ipynb', 'repo_name': 'migueltorrescosta/tutor', 'size': '3052'}