<jupyter_start><jupyter_text>Load the model<jupyter_code>from get_squeezenet import get_model
model, _, _ = get_model()
model.load_state_dict(torch.load('model.pytorch_state'))<jupyter_output><empty_output><jupyter_text>Error analysis  get all predictions and all misclassified images<jupyter_code>_, val_folder = get_image_folders()

val_iterator_no_shuffle = DataLoader(
    val_folder, batch_size=256, shuffle=False
)
val_predictions, val_true_targets,\
    erroneous_samples, erroneous_targets,\
    erroneous_predictions = predict(model, val_iterator_no_shuffle, return_erroneous=True)
# erroneous_samples: images that were misclassified
# erroneous_targets: their true labels
# erroneous_predictions: predictions for them<jupyter_output>100%|██████████| 40/40 [00:41<00:00,  1.13it/s]<jupyter_text>get human readable class names<jupyter_code>class_names = pd.read_csv('utils/words.txt', sep='\t', header=None)
class_names.columns = ['code', 'description']

decode = {}
for i, row in class_names.iterrows():
    decode[row.code] = row.description
    

# class index to human readable format
decode = {
    val_folder.class_to_idx[code]: decode[code] 
    for code in val_folder.class_to_idx
}<jupyter_output><empty_output><jupyter_text>number of misclassified images (there are overall 10k images in the val dataset)<jupyter_code>n_errors = len(erroneous_targets)
n_errors<jupyter_output><empty_output><jupyter_text>logloss and accuracies<jupyter_code>log_loss(val_true_targets, val_predictions)
accuracy_score(val_true_targets, val_predictions.argmax(1))
print(top_k_accuracy(val_true_targets, val_predictions, k=(2, 3, 4, 5, 10)))<jupyter_output>[0.72699999999999998, 0.7823, 0.81259999999999999, 0.83550000000000002, 0.90100000000000002]<jupyter_text>entropy of predictions<jupyter_code>hits = val_predictions.argmax(1) == val_true_targets
plt.hist(
    entropy(val_predictions[hits]), bins=30, 
    normed=True, alpha=0.7, label='correct prediction'
);
plt.hist(
    entropy(val_predictions[~hits]), bins=30, 
    normed=True, alpha=0.5, label='misclassification'
);
plt.legend();
plt.xlabel('entropy of predictions');<jupyter_output><empty_output><jupyter_text>confidence of predictions<jupyter_code>plt.hist(
    val_predictions[hits].max(1), bins=30, 
    normed=True, alpha=0.7, label='correct prediction'
);
plt.hist(
    val_predictions[~hits].max(1), bins=30, 
    normed=True, alpha=0.5, label='misclassification'
);
plt.legend();
plt.xlabel('confidence of predictions');<jupyter_output><empty_output><jupyter_text>difference between biggest and second biggest probability<jupyter_code>sorted_correct = np.sort(val_predictions[hits], 1)
sorted_incorrect = np.sort(val_predictions[~hits], 1)

plt.hist(
    sorted_correct[:, -1] - sorted_correct[:, -2], bins=30, 
    normed=True, alpha=0.7, label='correct prediction'
);
plt.hist(
    sorted_incorrect[:, -1] - sorted_incorrect[:, -2], bins=30, 
    normed=True, alpha=0.5, label='misclassification'
);
plt.legend();
plt.xlabel('difference');<jupyter_output><empty_output><jupyter_text>probabilistic calibration of the model<jupyter_code>model_calibration(val_true_targets, val_predictions, n_bins=10)<jupyter_output><empty_output><jupyter_text>per class accuracies<jupyter_code>per_class_acc = per_class_accuracy(val_true_targets, val_predictions)
plt.hist(per_class_acc);
plt.xlabel('accuracy');
most_inaccurate_k_classes(per_class_acc, 15, decode)<jupyter_output>plunger, plumber's helper 0.12
bow tie, bow-tie, bowtie 0.18
wooden spoon 0.18
syringe 0.18
dumbbell 0.2
bannister, banister, balustrade, balusters, handrail 0.24
ice cream, icecream 0.26
pole 0.3
punching bag, punch bag, punching ball, punchball 0.3
projectile, missile 0.3
cannon 0.32
frying pan, frypan, skillet 0.32
hog, pig, grunter, squealer, Sus scrofa 0.34
water jug 0.34
nail 0.36<jupyter_text>most confused pairs of classes<jupyter_code>confused_pairs = most_confused_classes(
    val_true_targets, val_predictions, decode, min_n_confusions=12
)
confused_pairs<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11973227565691621, 'fraction_numerical': 0.046355974219137334, 'mean_word_length': 7.733766233766234, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '535573', 'license': 'MIT', 'n_tokens_mistral': 1652, 'n_tokens_neox': 1460, 'n_words': 263, 'path': 'squeezenet/error_analysis.ipynb', 'repo_name': 'ikhlestov/nn-optimizations', 'size': '4034'}