<jupyter_start><jupyter_text>Right/Wrong Distinction<jupyter_code>err, kl_a, kl_r, kl_w, s_p, s_rp, s_wp = sess.run(
    [100 - acc, kl_all, kl_right, kl_wrong, s_prob, s_right_prob, s_wrong_prob],
    feed_dict={x: mnist.test.images, y: mnist.test.labels})

print('MNIST Error (%)| Prediction Prob (mean, std) | PProb Right (mean, std) | PProb Wrong (mean, std):')
print(err, '|', np.mean(s_p), np.std(s_p), '|', np.mean(s_rp), np.std(s_rp), '|', np.mean(s_wp), np.std(s_wp))

print('\nSuccess Detection')
print('Success base rate (%):', round(100-err,2))
print('KL[p||u]: Right/Wrong classification distinction')
safe, risky = kl_r, kl_w
labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
labels[:safe.shape[0]] += 1
examples = np.squeeze(np.vstack((safe, risky)))
print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))

print('Prediction Prob: Right/Wrong classification distinction')
safe, risky = s_rp, s_wp
labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
labels[:safe.shape[0]] += 1
examples = np.squeeze(np.vstack((safe, risky)))
print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))


print('\nError Detection')
print('Error base rate (%):', round(err,2))
safe, risky = -kl_r, -kl_w
labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
labels[safe.shape[0]:] += 1
examples = np.squeeze(np.vstack((safe, risky)))
print('KL[p||u]: Right/Wrong classification distinction')
print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))

print('Prediction Prob: Right/Wrong classification distinction')
safe, risky = -s_rp, -s_wp
labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
labels[safe.shape[0]:] += 1
examples = np.squeeze(np.vstack((safe, risky)))
print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))
def show_ood_detection_results(error_rate_for_in, in_examples, out_examples):
    kl_oos, s_p_oos = sess.run([kl_all, s_prob], feed_dict={x: out_examples})

    print('OOD Example Prediction Probability (mean, std):')
    print(np.mean(s_p_oos), np.std(s_p_oos))

    print('\nNormality Detection')
    print('Normality base rate (%):', round(100*in_examples.shape[0]/(
                out_examples.shape[0] + in_examples.shape[0]),2))
    print('KL[p||u]: Normality Detection')
    safe, risky = kl_a, kl_oos
    labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
    labels[:safe.shape[0]] += 1
    examples = np.squeeze(np.vstack((safe, risky)))
    print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
    print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))

    print('Prediction Prob: Normality Detection')
    safe, risky = s_p, s_p_oos
    labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
    labels[:safe.shape[0]] += 1
    examples = np.squeeze(np.vstack((safe, risky)))
    print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
    print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))

    print('Normality base rate (%):', round(100*(1 - err/100)*in_examples.shape[0]/
          (out_examples.shape[0] + (1 - err/100)*in_examples.shape[0]),2))
    print('KL[p||u]: Normality Detection (relative to correct examples)')
    safe, risky = kl_r, kl_oos
    labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
    labels[:safe.shape[0]] += 1
    examples = np.squeeze(np.vstack((safe, risky)))
    print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
    print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))

    print('Prediction Prob: Normality Detection (relative to correct examples)')
    safe, risky = s_rp, s_p_oos
    labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
    labels[:safe.shape[0]] += 1
    examples = np.squeeze(np.vstack((safe, risky)))
    print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
    print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))


    print('\n\nAbnormality Detection')
    print('Abnormality base rate (%):', round(100*out_examples.shape[0]/(
                out_examples.shape[0] + in_examples.shape[0]),2))
    print('KL[p||u]: Abnormality Detection')
    safe, risky = -kl_a, -kl_oos
    labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
    labels[safe.shape[0]:] += 1
    examples = np.squeeze(np.vstack((safe, risky)))
    print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
    print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))

    print('Prediction Prob: Abnormality Detection')
    safe, risky = -s_p, -s_p_oos
    labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
    labels[safe.shape[0]:] += 1
    examples = np.squeeze(np.vstack((safe, risky)))
    print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
    print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))

    print('Abnormality base rate (%):', round(100*out_examples.shape[0]/
          (out_examples.shape[0] + (1 - err/100)*in_examples.shape[0]),2))
    print('KL[p||u]: Abnormality Detection (relative to correct examples)')
    safe, risky = -kl_r, -kl_oos
    labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
    labels[safe.shape[0]:] += 1
    examples = np.squeeze(np.vstack((safe, risky)))
    print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
    print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))

    print('Prediction Prob: Abnormality Detection (relative to correct examples)')
    safe, risky = -s_rp, -s_p_oos
    labels = np.zeros((safe.shape[0] + risky.shape[0]), dtype=np.int32)
    labels[safe.shape[0]:] += 1
    examples = np.squeeze(np.vstack((safe, risky)))
    print('AUPR (%):', round(100*sk.average_precision_score(labels, examples), 2))
    print('AUROC (%):', round(100*sk.roc_auc_score(labels, examples), 2))<jupyter_output><empty_output><jupyter_text>Omniglot OOD Detection<jupyter_code>import scipy.io as sio
import scipy.misc as scimisc
safe_list = [0,2,5,6,8,12,13,14,15,16,17,18,19,21,26]  # other alphabets have characters which look like digits
m = sio.loadmat("./data/data_background.mat")

squished_set = []
for safe_number in safe_list:
    for alphabet in m['images'][safe_number]:
        for letters in alphabet:
            for letter in letters:
                for example in letter:
                    squished_set.append(scimisc.imresize(1 - example[0], (28,28)).reshape(1, 28*28))

safe_images = np.concatenate(squished_set, axis=0)
print('Omniglot\n')
show_ood_detection_results(err, mnist.test.images, safe_images)<jupyter_output>Omniglot

OOD Example Prediction Probability (mean, std):
0.866063 0.178644

Normality Detection
Normality base rate (%): 52.08
KL[p||u]: Normality Detection
AUPR (%): 96.28
AUROC (%): 95.61
Prediction Prob: Normality Detection
AUPR (%): 96.05
AUROC (%): 95.25
Normality base rate (%): 51.71
KL[p||u]: Normality Detection (relative to correct examples)
AUPR (%): 96.68
AUROC (%): 96.33
Prediction Prob: Normality Detection (relative to correct examples)
AUPR (%): 96.46
AUROC (%): 96.0


Abnormality Detection
Abnormality base rate (%): 47.92
KL[p||u]: Abnormality Detection
AUPR (%): 94.52
AUROC (%): 95.61
Prediction Prob: Abnormality Detection
AUPR (%): 94.02
AUROC (%): 95.25
Abnormality base rate (%): 48.29
KL[p||u]: Abnormality Detection (relative to correct examples)
AUPR (%): 95.91
AUROC (%): 96.33
Prediction Prob: Abnormality Detection (relative to correct examples)
AUPR (%): 95.61
AUROC (%): 96.0<jupyter_text>notMNIST OOS Detection<jupyter_code>pickle_file = './data/notMNIST.pickle'
with open(pickle_file, 'rb') as f:
    save = pickle.load(f, encoding='latin1')
    test_dataset = save['test_dataset'].reshape((-1, 28 * 28))
    del save  # hint to help gc free up memory

print('notMNIST\n')

show_ood_detection_results(err, mnist.test.images, test_dataset)<jupyter_output>notMNIST

OOD Example Prediction Probability (mean, std):
0.938411 0.127357

Normality Detection
Normality base rate (%): 50.0
KL[p||u]: Normality Detection
AUPR (%): 82.81
AUROC (%): 82.94
Prediction Prob: Normality Detection
AUPR (%): 84.47
AUROC (%): 82.87
Normality base rate (%): 49.63
KL[p||u]: Normality Detection (relative to correct examples)
AUPR (%): 83.18
AUROC (%): 83.78
Prediction Prob: Normality Detection (relative to correct examples)
AUPR (%): 84.86
AUROC (%): 83.71


Abnormality Detection
Abnormality base rate (%): 50.0
KL[p||u]: Abnormality Detection
AUPR (%): 84.48
AUROC (%): 82.94
Prediction Prob: Abnormality Detection
AUPR (%): 84.67
AUROC (%): 82.87
Abnormality base rate (%): 50.37
KL[p||u]: Abnormality Detection (relative to correct examples)
AUPR (%): 86.58
AUROC (%): 83.78
Prediction Prob: Abnormality Detection (relative to correct examples)
AUPR (%): 86.82
AUROC (%): 83.71<jupyter_text>CIFAR-10 OOD detection<jupyter_code>_, _, X_test, _ = load_data10()
cifar_batch = sess.run(tf.image.resize_images(tf.image.rgb_to_grayscale(X_test), 28, 28))

print('CIFAR-10bw\n')

show_ood_detection_results(err, mnist.test.images, cifar_batch.reshape(-1, 28*28))<jupyter_output>CIFAR-10bw

OOD Example Prediction Probability (mean, std):
0.88425 0.168702

Normality Detection
Normality base rate (%): 50.0
KL[p||u]: Normality Detection
AUPR (%): 94.21
AUROC (%): 94.15
Prediction Prob: Normality Detection
AUPR (%): 94.07
AUROC (%): 93.7
Normality base rate (%): 49.63
KL[p||u]: Normality Detection (relative to correct examples)
AUPR (%): 94.64
AUROC (%): 94.9
Prediction Prob: Normality Detection (relative to correct examples)
AUPR (%): 94.51
AUROC (%): 94.47


Abnormality Detection
Abnormality base rate (%): 50.0
KL[p||u]: Abnormality Detection
AUPR (%): 93.77
AUROC (%): 94.15
Prediction Prob: Abnormality Detection
AUPR (%): 93.23
AUROC (%): 93.7
Abnormality base rate (%): 50.37
KL[p||u]: Abnormality Detection (relative to correct examples)
AUPR (%): 95.16
AUROC (%): 94.9
Prediction Prob: Abnormality Detection (relative to correct examples)
AUPR (%): 94.83
AUROC (%): 94.47<jupyter_text>Sheer White Noise<jupyter_code>print('Sheer White Gaussian Noise\n')

show_ood_detection_results(err, mnist.test.images, np.random.normal(size=(10000, 28*28)))

# caveat: if you let the scale = 100 this will fail because it is inputting to much
# energy into the network, but this is unrealistic<jupyter_output>Sheer White Gaussian Noise

OOD Example Prediction Probability (mean, std):
0.917362 0.144497

Normality Detection
Normality base rate (%): 50.0
KL[p||u]: Normality Detection
AUPR (%): 88.19
AUROC (%): 88.59
Prediction Prob: Normality Detection
AUPR (%): 88.93
AUROC (%): 88.29
Normality base rate (%): 49.63
KL[p||u]: Normality Detection (relative to correct examples)
AUPR (%): 88.59
AUROC (%): 89.42
Prediction Prob: Normality Detection (relative to correct examples)
AUPR (%): 89.34
AUROC (%): 89.12


Abnormality Detection
Abnormality base rate (%): 50.0
KL[p||u]: Abnormality Detection
AUPR (%): 89.01
AUROC (%): 88.59
Prediction Prob: Abnormality Detection
AUPR (%): 88.9
AUROC (%): 88.29
Abnormality base rate (%): 50.37
KL[p||u]: Abnormality Detection (relative to correct examples)
AUPR (%): 90.86
AUROC (%): 89.42
Prediction Prob: Abnormality Detection (relative to correct examples)
AUPR (%): 90.83
AUROC (%): 89.12<jupyter_text>Sheer Uniform Noise (in the positive orthant)<jupyter_code>print('Sheer Uniform Noise\n')

show_ood_detection_results(err, mnist.test.images, np.random.uniform(size=(10000, 28*28)))<jupyter_output>Sheer Uniform Noise

OOD Example Prediction Probability (mean, std):
0.881227 0.15026

Normality Detection
Normality base rate (%): 50.0
KL[p||u]: Normality Detection
AUPR (%): 98.68
AUROC (%): 98.0
Prediction Prob: Normality Detection
AUPR (%): 98.63
AUROC (%): 97.89
Normality base rate (%): 49.63
KL[p||u]: Normality Detection (relative to correct examples)
AUPR (%): 99.1
AUROC (%): 98.7
Prediction Prob: Normality Detection (relative to correct examples)
AUPR (%): 99.06
AUROC (%): 98.63


Abnormality Detection
Abnormality base rate (%): 50.0
KL[p||u]: Abnormality Detection
AUPR (%): 96.5
AUROC (%): 98.0
Prediction Prob: Abnormality Detection
AUPR (%): 96.06
AUROC (%): 97.89
Abnormality base rate (%): 50.37
KL[p||u]: Abnormality Detection (relative to correct examples)
AUPR (%): 98.02
AUROC (%): 98.7
Prediction Prob: Abnormality Detection (relative to correct examples)
AUPR (%): 97.73
AUROC (%): 98.63
{'fraction_non_alphanumeric': 0.1702144188937228, 'fraction_numerical': 0.057799875699192045, 'mean_word_length': 5.215837759536456, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '419731', 'license': 'MIT', 'n_tokens_mistral': 5446, 'n_tokens_neox': 4661, 'n_words': 1081, 'path': 'Vision/MNIST_Softmax_Stats.ipynb', 'repo_name': 'hendrycks/error-detection', 'size': '12872'}