<jupyter_start><jupyter_text>Labelled Predictions Analysis<jupyter_code>import os
import json
import pandas as pd
import numpy as np

import config
import matplotlib.pyplot as plt
from pprint import pprint
from torchvision import transforms

import seaborn as sns
sns.set()
%matplotlib inline
attribute_names = ['5_o_Clock_Shadow', 'Arched_Eyebrows', 'Attractive', 'Bags_Under_Eyes', 'Bald', 
                       'Bangs', 'Big_Lips', 'Big_Nose', 'Black_Hair', 'Blond_Hair', 'Blurry', 'Brown_Hair', 
                       'Bushy_Eyebrows', 'Chubby', 'Double_Chin', 'Eyeglasses', 'Goatee', 'Gray_Hair',
                       'Heavy_Makeup', 'High_Cheekbones', 'Male', 'Mouth_Slightly_Open', 'Mustache', 
                       'Narrow_Eyes', 'No_Beard', 'Oval_Face', 'Pale_Skin', 'Pointy_Nose', 'Receding_Hairline',
                       'Rosy_Cheeks', 'Sideburns', 'Smiling', 'Straight_Hair', 'Wavy_Hair', 'Wearing_Earrings', 
                       'Wearing_Hat', 'Wearing_Lipstick', 'Wearing_Necklace', 'Wearing_Necktie', 'Young']
def read_pred_accuracy_json(tpfile):
    all_accs = pd.DataFrame(index = attribute_names)
    all_accs.index.name = "Attributes"
    for dirt in os.listdir(config.INFERENCE_DIR):
        dirpath = os.path.join(config.INFERENCE_DIR, dirt)
        if os.path.isdir(dirpath): 
            for run in os.listdir(dirpath):
                runpath = os.path.join(dirpath, run)
                if os.path.isdir(runpath): 
                    for filename in os.listdir(runpath):
                        if filename == tpfile:
                            tpfile_path = os.path.join(runpath,tpfile)
                            tpjson = json.load(open(tpfile_path))
                            acc = pd.DataFrame.from_dict(tpjson, orient='index', columns=[run])
                            acc.index.name = "Attributes"
                            acc = acc.round(decimals=2)
                            all_accs = all_accs.merge(acc, left_index=True, right_on="Attributes")
    return all_accs
                        
    
all_test_accs = read_pred_accuracy_json("test_preds.json") # val_preds.json
# for report 
all_test_accs2 = all_test_accs.transpose()
with pd.option_context('display.max_rows', None, 'display.max_columns', None):  
    display(all_test_accs2.loc[:,'5_o_Clock_Shadow':'High_Cheekbones'])
    display(all_test_accs2.loc[:,'Male':'Young'])
f = plt.figure()
plt.title('Labelled Test Scores Comparison', color='black')
all_test_accs.plot(kind='barh', figsize=(15, 15), ax=f.gca())
# for p in f.gca().patches:
#     value = round(p.get_width(),2)
#     f.gca().annotate(str(value), xy=(p.get_width(), p.get_y()))
plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
plt.show()
all_val_accs = read_pred_accuracy_json("val_preds.json") # val_preds.json
# for report 
all_val_accs2 = all_val_accs.transpose()
with pd.option_context('display.max_rows', None, 'display.max_columns', None):  
    display(all_val_accs2.loc[:,'5_o_Clock_Shadow':'High_Cheekbones'])
    display(all_val_accs2.loc[:,'Male':'Young'])
f = plt.figure()
plt.title('Labelled Val Scores Comparison', color='black')
all_val_accs.plot(kind='barh', figsize=(20, 15), ax=f.gca())
# for p in f.gca().patches:
#     value = round(p.get_width(),2)
#     f.gca().annotate(str(value), xy=(p.get_width(), p.get_y()))
plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
plt.show()<jupyter_output><empty_output><jupyter_text>Select the best model prediction based on all high scoring models (NOT IN USE)+ Inspired by Ensemble methods, BUT NOT USING ENSEMBLE PREDICTIONS FOR SUBMISSION!+ Create a most commonly predicted attributes dataframe based on median of all model predictions+ Compare all model predictions differences between most common predicted and choose the predictions with the lowest differences i.e. generally predicted the same attributes as the other models.+ Requires 3+ good models!!<jupyter_code>pfile = "predictions.csv"
all_preds = []
for dirt in os.listdir(config.INFERENCE_DIR):
    dirpath = os.path.join(config.INFERENCE_DIR, dirt)
    if os.path.isdir(dirpath): 
        for filename in os.listdir(dirpath):
            if filename == pfile:
                pfile_path = os.path.join(dirpath,pfile)
                pprint(pfile_path)
                pred = pd.read_csv(pfile_path, index_col=0)
                all_preds.append((pred, dirt))
                #print(pred)
print(f"{len(all_preds)} predictions loaded.")
all_preds[1][0]
def get_most_common_predictions(all_preds):
    most_common_pred = pd.DataFrame(index=all_preds[0][0].index)

    for column in attribute_names:
        column_data = []
        for i in range(len(all_preds)):
            column_data.append(all_preds[i][0][column])
        most_common_pred[column] = np.median(column_data, axis=0)
    return most_common_pred

most_common_pred = get_most_common_predictions(all_preds)
def get_highest_scoring_prediction_id(all_preds, most_common_pred):
    pred_diffs = {}
    for mpred, model in all_preds:
        diff_df = most_common_pred.compare(mpred, align_axis=0)
        pred_diffs[model] = np.sum(diff_df.count()) // 2
    pprint(pred_diffs)
    return min(pred_diffs, key=pred_diffs.get)

pprint(f"Most common predictions model: {get_highest_scoring_prediction_id(all_preds, most_common_pred)}")<jupyter_output>{'FaceAttrMobileNetV2_50_42_s0e80_tb100_vb100_MU_FL_SGD_53_lr0.01_wd0.0001_ReduceLROnPlateau': 20047,
 'FaceAttrResNeXt_50_42_s0e80_tb128_vb128_MU_FL_SGD_52_lr0.01_wd0.0001_ReduceLROnPlateau': 20047}
('Most common predictions model: '
 'FaceAttrMobileNetV2_50_42_s0e80_tb100_vb100_MU_FL_SGD_53_lr0.01_wd0.0001_ReduceLROnPlateau')<jupyter_text>Prediction Analysis<jupyter_code>model_idx =0
preds = all_preds[model_idx][0]
stat_df = pd.DataFrame(index = attribute_names)
stat_df.loc[:,'Testset'] = (preds.iloc[:,:] == 1).mean(axis=0)*100
stat_df = stat_df.sort_values('Testset', ascending=False)
fig, ax = plt.subplots()
stat_df.plot(title='CelebA Private Testset Prediction Frequency Distribution', 
             kind='bar', figsize=(20, 5), ax=ax, color='green')
for p in ax.patches:
    value = round(p.get_height(),2)
    ax.annotate(str(value), xy=(p.get_x(), p.get_height()))
plt.savefig('private_test.png',dpi=160, bbox_inches='tight')
print(preds[(preds['Young']==1) & (preds['Gray_Hair']==1)].index)
print(preds[(preds['Male']==-1) & (preds['Mustache']==1)].index)
print(preds[(preds['Male']==-1) & (preds['Goatee']==1)].index)
print(preds[(preds['Gray_Hair']==1) & (preds['Blond_Hair']==1)].index)
print(preds[(preds['Male']==-1) & (preds['No_Beard']==-1)].index)
print(preds[(preds['Rosy_Cheeks']==1) & (preds['Rosy_Cheeks']==1)].index)
print(len(preds[(preds['Rosy_Cheeks']==1) & (preds['Rosy_Cheeks']==1)].index))
print(preds[(preds['Wearing_Necklace']==1) & (preds['Wearing_Necklace']==1)].index)
print(len(preds[(preds['Wearing_Necklace']==1) & (preds['Wearing_Necklace']==1)].index))
inv_normalize = transforms.Normalize(
   mean=[-0.485/0.229, -0.456/0.224, -0.406/0.225],
   std=[1/0.229, 1/0.224, 1/0.225]
)

def get_celeb_prediction(preds, name, first_img=False):
    celeb_preds = preds[preds.index.str.contains(name)]
    celeb_first = celeb_preds.index[0]
    celeb_stat = pd.DataFrame(index = attribute_names)
    celeb_stat.loc[:,name] = (celeb_preds.iloc[:,:] == 1).mean(axis=0)*100
    mycolor = 'skyblue' if celeb_stat.loc['Male',name] >= 50 else 'magenta'
    celeb_stat = celeb_stat.sort_values(name, ascending=False)
    ncols = 3 if first_img else 2
    ax = plt.subplot2grid((1, ncols), (0, 0), colspan=2)
    celeb_stat.plot(title=name+' Prediction Frequency Distribution', 
                 kind='bar', figsize=(20, 5), color=mycolor, ax=ax)
    for p in ax.patches:
        value = round(p.get_height(),2)
        ax.annotate(str(value), xy=(p.get_x(), p.get_height()))
    if first_img:
        ax2 = plt.subplot2grid((1, ncols), (0, 2), colspan=1)
        index = test_dataset.imagenames.index(celeb_first)    
        s_img = inv_normalize(test_dataset[index][0]).permute(1, 2, 0)
        ax2.imshow(s_img)
        ax2.set_axis_off()
        plt.title(celeb_first)
        plt.tight_layout()
    plt.show()
get_celeb_prediction(preds, name = 'George_W_Bush') # Male, Gray_Hair, Blonde, Necktie<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14673585133341377, 'fraction_numerical': 0.026668275612404972, 'mean_word_length': 4.078431372549019, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '572668', 'license': 'MIT', 'n_tokens_mistral': 3244, 'n_tokens_neox': 3044, 'n_words': 502, 'path': 'project_1/src/P_Predictions Analysis.ipynb', 'repo_name': 'kkaryl/AI6126-Advanced_Computer_Vision', 'size': '8287'}