<jupyter_start><jupyter_text>FER Facial Expression Recognition <NAME> :), :(, :/  Metric for test<jupyter_code>import numpy as np
import pandas as pd
import os

# from fer import visualizationutl as vz

import matplotlib.pyplot as plt
%matplotlib inline
PROJECT='./netruns'
EXP_NAME='exp_net_alexnet_lr0001pl_daum_proc_001'
filenametrain = os.path.join( PROJECT, EXP_NAME, 'train.csv' )
filenameval = os.path.join( PROJECT, EXP_NAME, 'val.csv' )
print(filenametrain)
print(filenameval)<jupyter_output>./netruns/exp_net_alexnet_lr0001pl_daum_proc_001/train.csv
./netruns/exp_net_alexnet_lr0001pl_daum_proc_001/val.csv<jupyter_text>Train<jupyter_code>train = pd.read_csv(filenametrain)
train.head()

indx = train['type']
xiter = np.array(train['iter'][indx])
xtop1 = np.array(train['top1'][indx])
xtop2 = np.array(train['top2'][indx])
xloss = np.array(train['loss'][indx])

#plt.plot( xtop1 )
#plt.plot( xtop2 )
#plt.show()


#plt.style.use('default')
#plt.style.use('classic')
#plt.style.use('ggplot')
#plt.style.use('fivethirtyeight')
#plt.style.use('dark_background')
#plt.style.context('Solarize_Light2')

_, ax1 = plt.subplots()
ax2 = ax1.twinx()
pl1 = ax1.plot(np.arange(len(xloss)), xloss, label='loss')
pl2 = ax2.plot(np.arange(len(xloss)), xtop1, 'r', label='top1')
pl3 = ax2.plot(np.arange(len(xloss)), xtop2, 'g', label='top2')

ax1.set_xlabel('iteration')
ax1.set_ylabel('train loss')
ax2.set_ylabel('test accuracy')
ax2.set_title('Test Accuracy: {:.2f}'.format( xtop1[-1] ) )

#ax2.legend(loc='right')
lns = pl1+pl2+pl3
labs = [l.get_label() for l in lns]
ax1.legend(lns, labs, loc='right')

ax1.grid(True)


plt.plot( xloss )
plt.show()<jupyter_output><empty_output><jupyter_text>Validation<jupyter_code>val = pd.read_csv(filenameval)
val.head()

xtop1 = val['top1']
xtop2 = val['top2']

plt.plot( xtop1 )
plt.plot( xtop2 )
plt.show()<jupyter_output><empty_output><jupyter_text>Experimento Zoo pytorch for FER<jupyter_code>PROJECT='./netruns'
filename = 'experiments.txt'

filetrain = list()
fileval = list()
names = list()

with open(filename, 'r') as file:
    for name in file:          
        name = name.split()[0]
        namenet = name.split('_')[2]        
        filenametrain = os.path.join( PROJECT, name, 'train.csv' )
        filenameval = os.path.join( PROJECT, name, 'val.csv' )         
        filetrain.append(filenametrain)
        fileval.append(filenameval)
        names.append(namenet)


print(filetrain)
print(fileval)
print(names)


def smooth(x,window_len=11,window='hanning'):    
    """
    http://scipy-cookbook.readthedocs.io/items/SignalSmooth.html
    """
    s=np.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]]
    #print(len(s))
    if window == 'flat': #moving average
        w=np.ones(window_len,'d')
    else:
        w=eval('np.'+window+'(window_len)')
    y=np.convolve(w/w.sum(),s,mode='valid')
    return y

cmap = plt.get_cmap('jet_r')
windows=['flat', 'hanning', 'hamming', 'bartlett', 'blackman']

plt.figure( figsize=(8,6) )
for i,file in enumerate(filetrain):   

    train = pd.read_csv(file)      
    indx = train['type']
    xepoch = np.array(train['epoch'][indx])
    xiter = np.array(train['iter'][indx])
    xtop1 = np.array(train['top1'][indx])
    xtop2 = np.array(train['top2'][indx])
    xloss = np.array(train['loss'][indx])
    
    xftop1 = smooth(xtop1,25,windows[3]) 
    xftop1 = xftop1[:len(xtop1)]
         
    color = cmap(float(i)/len(filetrain))
    plt.plot(xepoch, xtop1, alpha=0.2, c=color)   
    plt.plot(xepoch, xftop1,label='{}'.format( names[i] ), c=color)
    
    
plt.xlabel('epochs')
plt.ylabel('top1')
plt.title('Training Top@1 (Accuracy) for Ferp dataset')
    
plt.grid(True) 
plt.legend()
#plt.savefig('./netruns/top1.png')
plt.show()


plt.figure( figsize=(8,6) )
for i,file in enumerate(filetrain):   

    train = pd.read_csv(file)      
    indx = train['type']
    xepoch = np.array(train['epoch'][indx])
    xiter = np.array(train['iter'][indx])
    xtop1 = np.array(train['top1'][indx])
    xtop2 = np.array(train['top2'][indx])
    xloss = np.array(train['loss'][indx])
    
    xftop2 = smooth(xtop2,25,windows[3]) 
    xftop2 = xftop2[:len(xtop2)]
         
    color = cmap(float(i)/len(filetrain))
    plt.plot(xepoch, xtop2, alpha=0.2, c=color)   
    plt.plot(xepoch, xftop2,label='{}'.format( names[i] ), c=color)
    
    
plt.xlabel('epochs')
plt.ylabel('top2')
plt.title('Training Top@2 (Accuracy) for Ferp dataset')
    
plt.grid(True) 
plt.legend()
#plt.savefig('./netruns/top1.png')
plt.show()

plt.figure( figsize=(8,6) )
for i,file in enumerate(filetrain):   

    train = pd.read_csv(file)      
    indx = train['type']
    xepoch = np.array(train['epoch'][indx])
    xiter = np.array(train['iter'][indx])
    xtop1 = np.array(train['top1'][indx])
    xtop2 = np.array(train['top2'][indx])
    xloss = np.array(train['loss'][indx])
    
    xfloss = smooth(xloss,25,windows[3]) 
    xfloss = xfloss[:len(xloss)]
         
    color = cmap(float(i)/len(filetrain))
    plt.plot(xepoch, xloss, alpha=0.2, c=color)   
    plt.plot(xepoch, xfloss,label='{}'.format( names[i] ), c=color)
    
    
plt.xlabel('epochs')
plt.ylabel('top2')
plt.title('Training loss for Ferp dataset')
    
plt.grid(True) 
plt.legend()
#plt.savefig('./netruns/top1.png')
plt.show()




_, ax1 = plt.subplots( figsize=(8, 6) )
ax2 = ax1.twinx()


for i,file in enumerate(filetrain):   

    train = pd.read_csv(file)      
    indx = train['type']
    xepoch = np.array(train['epoch'][indx])
    xiter = np.array(train['iter'][indx])
    xtop1 = np.array(train['top1'][indx])
    xtop2 = np.array(train['top2'][indx])
    xloss = np.array(train['loss'][indx])
    
    xfloss = smooth(xloss,25,windows[3]) 
    xfloss = xfloss[:len(xloss)]
    xftop1 = smooth(xtop1,25,windows[3]) 
    xftop1 = xftop1[:len(xtop1)]
         
    color = cmap(float(i)/len(filetrain))
            
    ax1.plot(xepoch, xloss, c=color, alpha=0.2 )
    pl1 = ax1.plot(xepoch, xfloss, label='{}'.format( names[i] ), c=color)      
    
    ax2.plot(xepoch, xtop1, c=color, alpha=0.2 )
    pl2 = ax2.plot(xepoch, xftop1, label='{}'.format( names[i] ), c=color) 
    

ax1.set_xlabel('epochs')
ax1.set_ylabel('train loss')
ax2.set_ylabel('test accuracy')
ax2.set_title('Training Top@1 (Accuracy) for Ferp dataset' )


ax1.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=2.5)
#lns = pl1+pl2
#labs = [l.get_label() for l in lns]
#ax1.legend(lns, labs, loc='right')

ax1.grid(True)
    
    
#plt.xlabel('epochs')
#plt.ylabel('top2')
#plt.title('Training Top@2 (Accuracy) for Ferp dataset')
    
#plt.grid(True) 
#plt.legend()
#plt.savefig('./netruns/top1.png')

plt.show()


plt.figure( figsize=(8,6) )
cmap = plt.get_cmap('jet_r')

for i,file in enumerate(fileval):
    
    val = pd.read_csv(file)   
    xtop1 = np.array(val['top1'])
    xtop2 = np.array(val['top2'])   
    
    color = cmap(float(i)/len(fileval))
    plt.plot(xtop1, label='{}'.format( names[i] ), c=color )

    
plt.xlabel('epochs')
plt.ylabel('top1')
plt.title('Validate Top@1 (Accuracy) for Ferp dataset')

plt.legend()
plt.grid(True)
#plt.savefig('./netruns/top1.png')
plt.show()


cmap = plt.get_cmap('jet_r')
windows=['flat', 'hanning', 'hamming', 'bartlett', 'blackman']
namesex = ['VGG11', 'SimpleNet', 'ResNet18', 'InceptionV3', 'FMPNet', 'DeXpression', 'CVGG', 'AlexNet', 'DenseNet121']

_, ax1 = plt.subplots( figsize=(8, 6) )
ax2 = ax1.twinx()

for i,file in enumerate(filetrain):   

    train = pd.read_csv(file)      
    indx = train['type']
    xepoch = np.array(train['epoch'][indx])
    xiter = np.array(train['iter'][indx])
    xloss = np.array(train['loss'][indx])   
    
    xfloss = vz.smooth(xloss,25,windows[3]) 
    xfloss = xfloss[:len(xloss)]     
    
    color = cmap(float(i)/len(filetrain))            
    ax1.plot(xepoch, xloss, c=color, alpha=0.2 )
    pl1 = ax1.plot(xepoch, xfloss, label='{}'.format( namesex[i] ), c=color)   
    


for i,file in enumerate(fileval):
    
    val = pd.read_csv(file)   
    xtop1 = np.array(val['top1'])
    xtop1 = np.array(val['top2'])      
    color = cmap(float(i)/len(fileval))
    ax2.plot(xtop1, label='{}'.format( namesex[i] ), c=color )  
    

ax1.set_xlabel('Epochs')
ax1.set_ylabel('Train loss')
ax2.set_ylabel('Valid Top-2 accuracy (%)')
#ax2.set_title('Results of Training in the Ferp dataset' )

ax1.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=2.5)
ax1.grid(True)
#plt.savefig('./netruns/top1.png')
plt.show()<jupyter_output><empty_output><jupyter_text>Metrics<jupyter_code>import sklearn.metrics as metrics


#exp_net_vgg11_lr0001pl_daum_proc_001
#exp_net_simplenet_lr0001pl_daum_proc_001
#exp_net_resnet18_lr0001pl_daum_proc_001
#exp_net_inceptionv3_lr0001pl_daum_proc_001
#exp_net_fmp_lr0001pl_daum_proc_001
#exp_net_dexpression_lr0001pl_daum_proc_001
#exp_net_cvgg_lr0001pl_daum_proc_001
#exp_net_alexnet_lr0001pl_daum_proc_001
#exp_net_densenet121_lr0001pl_daum_proc_001


mdata = pd.read_csv('./netruns/exp_net_inceptionv3_lr0001pl_daum_proc_001/jaffe_test.csv')
data = mdata.as_matrix()

yhat = np.argmax( data[:,:-1], axis=1 )
y = data[:,-1]

print(data.shape)
print(y.shape)
acc = metrics.accuracy_score(y, yhat)
precision = metrics.precision_score(y, yhat, average='macro')
recall = metrics.recall_score(y, yhat, average='macro')
f1_score = 2*precision*recall/(precision+recall)

print('Accuracy : %f' % acc)
print('Precision : %f' % precision)
print('Recall : %f' % recall)
print('F1 score : %f' % f1_score)
print("")
metric = metrics.classification_report(y, yhat)
print(metric)
mc = metrics.confusion_matrix(y, yhat)
print(mc)
import itertools
emotion_name = ['neutral', 'happiness', 'surprise', 'sadness', 'anger', 'disgust', 'fear', 'contempt']
def plot_confusion_matrix(cm, classes,
                          normalize=True,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    #print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    #plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')


plt.figure( figsize=(8,8))
plot_confusion_matrix(mc, classes=emotion_name, title=' ')
plt.show()

PROJECT='./netruns'
filename = 'experiments.txt'

files  = list()
names = list()

with open(filename, 'r') as file:
    for name in file:          
        name = name.split()[0]
        namenet = name.split('_')[2]     
        
        filenameferp = os.path.join( PROJECT, name, 'test.csv' )       
        filenameck = os.path.join( PROJECT, name, 'ck_test.csv' )
        filenamejaffe = os.path.join( PROJECT, name, 'jaffe_test.csv' )
        filenamebu = os.path.join( PROJECT, name, 'bu_test.csv' )  
        
        files.append([filenameferp, filenameck, filenamejaffe,  filenamebu])        
        names.append(namenet)
    

print(names)
print(files)
import sklearn.metrics as fmetrics


datasets = ['ferp', 'ckp', 'jaffe', 'bu']
metrics = list()

for i, name in enumerate(names):
    #print(i,name)
    
    tupla = list()
    for j,file in enumerate(files[i]):
        #print(datasets[j], file)   
    
        mdata = pd.read_csv(file)        
        data = mdata.as_matrix()
        yhat = np.argmax( data[:,:-1], axis=1 )
        y = data[:,-1]        

        acc = fmetrics.accuracy_score(y, yhat)        
        precision = fmetrics.precision_score(y, yhat, average='macro')
        recall = fmetrics.recall_score(y, yhat, average='macro')
        f1_score = 2*precision*recall/(precision+recall)
        
        metric = { 
            'Accuracy': acc,
            'Precision': precision,
            'Recall': recall,
            'F1score': f1_score,          
            }        
        
        tupla.append( metric )        
        
        #print('Accuracy : %f' % acc)
        #print('Precision : %f' % precision)
        #print('Recall : %f' % recall)
        #print('F1 score : %f' % f1_score)
        #print("")
        
    metrics.append( {name:tupla} )
    

print(metrics)
    


data = list()
datasets = ['ferp', 'ckp', 'jaffe', 'bu']
idataset = 0

for mth in metrics:
    for k,v in mth.items():
        v = v[idataset]
        #print(k,v['Recall'], v['Precision'], v['F1score'] )
        #data.append( [v['Recall'], v['Precision'], v['F1score']] )
        data.append(v)
        
#data = np.array(data)
#print(data)

print(datasets[idataset])
df = pd.DataFrame( data, index=namesex )
print(df)
#df.to_csv( os.path.join( './netruns', 'metrics_{}.csv'.format(datasets[idataset]) ), index=False, encoding='utf-8')


#df.plot.bar( figsize=(8,6) )
ax = df.plot(kind="bar", figsize=(8,4),  stacked=False)
ax.grid('on', linestyle='-', linewidth=0.25)
ax.legend(bbox_to_anchor=(1.2, 0.5))
ax.set_ylabel('Metric value')
#ax.set_title('Metric for {} dataset'.format( datasets[idataset]) )

cmap = plt.get_cmap('jet_r')



namesex =  ['ResNet18',  'FMPNet', 'CVGG', 'AlexNet', 'PreActResNet18']
acc = np.array([ 87.695, 79.535, 84.316, 86.038, 82.372 ])
sizearch = np.array( [ 11190216,  6075400,  8757704, 57036616, 11170120])

sa = sizearch/1000000 #np.array([128, 1, 11, 21,  6, 0.8, 8, 57, 7])
# sc = np.array([20, 5, 10, 5, 17  ])
sc = np.array([ 106, 28, 50, 107, 53  ])

# mem = { 'alexnet':107.96,
#         'preactresnet18': 53.37,
#         'cvgg13': 50.16,
#         'fmp': 28.0,
#         'resnet18': 106.01
        
#       }

# acc = np.array(df['Accuracy'])
# sizearch = np.array([128799112, 1740580, 11190216, 21836392,  6075400, 846024, 8757704, 57036616, 7045704])
# sa = sizearch/1000000 #np.array([128, 1, 11, 21,  6, 0.8, 8, 57, 7])
# sc = np.array([8, 2, 20, 94, 5, 7, 10, 5, 120  ])


plt.figure(figsize=(8,8))
#fig, ax = plt.subplots()
ax=plt.subplot(aspect='equal')

for i in range(5):
    color = cmap(float(i)/len(sa))
    circ = plt.Circle( (sc[i], acc[i]), sa[i]/np.sum(sa)*60,  alpha=0.6, lw=2, edgecolor=color, facecolor=color, label='{}'.format( namesex[i] ) )
    ax.add_patch(circ)
    circ = plt.Circle( (sc[i],acc[i]), 0.1, edgecolor=color, facecolor='w' )
    ax.add_patch(circ)


ax.grid(True)    

plt.xlim(-1,140)
plt.ylim(0,120)
plt.xlabel('Estimated Total Size (MB)')
plt.ylabel('Accuracy (%)')
plt.legend()

plt.savefig('../out/cnnsize.eps')
plt.show()



tablatop1 = [
72.9936294555664, 30.0, 38.88888931274414,30.0, 85.50955200195312,41.904762268066406,58.611114501953125,43.965518951416016,
85.9738540649414,37.07317352294922,58.05555725097656,41.2068977355957,86.83875274658203,40.0,59.4444465637207,41.72414016723633,
87.8546371459961,41.463417053222656,60.27778244018555,45.0,87.47209930419922,38.04878234863281,61.388893127441406,43.103450775146484,
85.37285614013672,40.9523811340332,58.888893127441406,46.2068977355957,80.38216400146484,30.4761905670166,41.66666793823242,
35.86206817626953,84.26751708984375,42.85714340209961,58.888893127441406,43.620689392089844,
]


tablatop2 = [
87.9936294555664,39.52381134033203,57.77777862548828,37.413795471191406,96.33757781982422,56.66666793823242,70.27777862548828,
61.72414016723633,95.9834213256836,56.585365295410156,73.33333587646484,61.2068977355957,96.27151489257812,59.52381134033203,
72.5,62.413795471191406,96.11093139648438,57.560977935791016,76.3888931274414,61.72414016723633,96.33407592773438,
60.97561264038086,79.72222900390625,64.82759094238281,95.76163482666016,52.380950927734375,80.55555725097656,66.03448486328125,
92.9936294555664,51.42856979370117,59.722225189208984,50.86206817626953,95.15924072265625,58.095237731933594,76.11111450195312,
64.6551742553711,
]


tabla = np.array(tablatop2).reshape(-1,4)
men = (np.mean(tabla, axis=1))
sd  = (np.std(tabla, axis=1))

for i in range( len(tabla) ):
    print(i+1,'{:.3f} + {:.3f}'.format(men[i],sd[i]))<jupyter_output>1 55.677 + 20.268
2 71.252 + 15.279
3 71.777 + 15.255
4 72.677 + 14.449
5 72.946 + 15.092
6 75.465 + 13.935
7 73.683 + 16.178
8 63.752 + 17.243
9 73.505 + 14.067
{'fraction_non_alphanumeric': 0.1751548313390656, 'fraction_numerical': 0.10853225903433347, 'mean_word_length': 3.78205865439908, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '142440', 'license': 'MIT', 'n_tokens_mistral': 7973, 'n_tokens_neox': 6873, 'n_words': 1063, 'path': 'books/results_analysis.ipynb', 'repo_name': 'pedrodiamel/pytorch-classification', 'size': '16651'}