<jupyter_start><jupyter_text>counter-intuitive behavior when all the predictions are one-sided (>0.5, or <0.5) and the label prior is less than PCC<jupyter_code>import numpy as np
import freq_e
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
#set-up: all the test pred probs are <0.5, training prev also <0.5
#we get weird predictions when the training prev is less than 
#the mean of the test_pred_probs 
# I think this has to do with the ratios p(y=1|x)/pi and p(y=0|x)/(1-pi)
LABEL_PRIOR = 0.25
test_pred_probs = np.array([0.2, 0.3, 0.4])
log_odds = freq_e.estimate.calc_log_odds(test_pred_probs)
log_post_probs = freq_e.estimate.mll_curve(log_odds, LABEL_PRIOR)
print('LABEL_PRIOR=', LABEL_PRIOR)
print('PCC=', np.mean(test_pred_probs))
print('point=', freq_e.estimate.generative_get_map_est(log_post_probs))
plt.plot(freq_e.estimate.DEFAULT_THETA_GRID, log_post_probs)
plt.show()
#if we set the label prior equal to PCC or greater than we're fine 
LABEL_PRIOR = 0.3
log_post_probs = freq_e.estimate.mll_curve(log_odds, LABEL_PRIOR)
print('LABEL_PRIOR=', LABEL_PRIOR)
print('point=', freq_e.estimate.generative_get_map_est(log_post_probs))
plt.plot(freq_e.estimate.DEFAULT_THETA_GRID, log_post_probs)
plt.show()
LABEL_PRIOR = 0.5
log_post_probs = freq_e.estimate.mll_curve(log_odds, LABEL_PRIOR)
print('LABEL_PRIOR=', LABEL_PRIOR)
print('point=', freq_e.estimate.generative_get_map_est(log_post_probs))
plt.plot(freq_e.estimate.DEFAULT_THETA_GRID, log_post_probs)
plt.show()
LABEL_PRIOR = 0.9
log_post_probs = freq_e.estimate.mll_curve(log_odds, LABEL_PRIOR)
print('LABEL_PRIOR=', LABEL_PRIOR)
print('point=', freq_e.estimate.generative_get_map_est(log_post_probs))
plt.plot(freq_e.estimate.DEFAULT_THETA_GRID, log_post_probs)
plt.show()
#this same phenomena holds even with an extremely large number of docs 
LABEL_PRIOR = 0.25
test_pred_probs = np.random.uniform(low=0.1, high=0.5, size=10**6)
log_odds = freq_e.estimate.calc_log_odds(test_pred_probs)
log_post_probs = freq_e.estimate.mll_curve(log_odds, LABEL_PRIOR)
print('LABEL_PRIOR=', LABEL_PRIOR)
print('PCC=', np.mean(test_pred_probs))
print('point=', freq_e.estimate.generative_get_map_est(log_post_probs))
plt.plot(freq_e.estimate.DEFAULT_THETA_GRID, log_post_probs)
plt.show()<jupyter_output>LABEL_PRIOR= 0.25
PCC= 0.30017845418213585
point= 0.984<jupyter_text>for test pred probs that are centered around 0.5, the label prior completely shifts predictions in the opposite way as the label prior. does this "label prior" still make sense even when you've trained a classifier with a balanced objective function (takes into account class imbalance?)<jupyter_code>LABEL_PRIOR = 0.5
test_pred_probs = np.random.uniform(low=0.4, high=0.6, size=10**6)
log_odds = freq_e.estimate.calc_log_odds(test_pred_probs)
log_post_probs = freq_e.estimate.mll_curve(log_odds, LABEL_PRIOR)
print('LABEL_PRIOR=', LABEL_PRIOR)
print('PCC=', np.mean(test_pred_probs))
print('point=', freq_e.estimate.generative_get_map_est(log_post_probs))
plt.plot(freq_e.estimate.DEFAULT_THETA_GRID, log_post_probs)
plt.show()
LABEL_PRIOR = 0.2
log_odds = freq_e.estimate.calc_log_odds(test_pred_probs)
log_post_probs = freq_e.estimate.mll_curve(log_odds, LABEL_PRIOR)
print('LABEL_PRIOR=', LABEL_PRIOR)
print('PCC=', np.mean(test_pred_probs))
print('point=', freq_e.estimate.generative_get_map_est(log_post_probs))
plt.plot(freq_e.estimate.DEFAULT_THETA_GRID, log_post_probs)
plt.show()
LABEL_PRIOR = 0.8
log_odds = freq_e.estimate.calc_log_odds(test_pred_probs)
log_post_probs = freq_e.estimate.mll_curve(log_odds, LABEL_PRIOR)
print('LABEL_PRIOR=', LABEL_PRIOR)
print('PCC=', np.mean(test_pred_probs))
print('point=', freq_e.estimate.generative_get_map_est(log_post_probs))
plt.plot(freq_e.estimate.DEFAULT_THETA_GRID, log_post_probs)
plt.show()<jupyter_output>LABEL_PRIOR= 0.8
PCC= 0.5000329078706477
point= 0.0
{'fraction_non_alphanumeric': 0.11539456662354464, 'fraction_numerical': 0.025097024579560156, 'mean_word_length': 8.053864168618267, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '665709', 'license': 'MIT', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1603, 'n_words': 280, 'path': 'tests/sens_label_prior.ipynb', 'repo_name': 'slanglab/freq-e', 'size': '3865'}