<jupyter_start><jupyter_text>CONFIGURATE<jupyter_code>!ls ../out/
project          = '../out'
name             = 'ferattpaperstn_attgmmnet_ferattentiongmm_attloss_adam_affectnetdark_dim64_preactresnet18x32_fold0_001'
pathnamedataset  = '~/.datasets'
pathmodel        = os.path.join( project, name, 'models/model_best.pth.tar' ) #model_best
pathproject      = os.path.join( project, name )
batch_size       = 1
workers          = 1
cuda             = False
parallel         = False
gpu              = 1
seed             = 1
imsize           = 128<jupyter_output><empty_output><jupyter_text>LOAD MODEL<jupyter_code># load model
print('>> Load model ...')

net = AttentionGMMNeuralNet( 
    patchproject=project, 
    nameproject=name, 
    no_cuda=cuda, 
    parallel=parallel, 
    seed=seed, 
    gpu=gpu 
    )

if net.load( pathmodel ) is not True:
    assert(False)<jupyter_output>WARNING:root:Setting up a new session...
WARNING:visdom:Without the incoming socket you cannot receive events from the server or register event handlers to your Visdom client.<jupyter_text>DATASETS<jupyter_code>print('>> Load dataset ...')
namedataset = FactoryDataset.affect
subset = FactoryDataset.validation
imagesize=128

dataset = Dataset(    
    data=FactoryDataset.factory(
        pathname=pathnamedataset, 
        name=namedataset, 
        subset=subset, 
        download=True 
    ),
    num_channels=3,
    transform=transforms.Compose([
            mtrans.ToResize( (imagesize,imagesize), resize_mode='square' ),
            mtrans.ToTensor(),
            #mtrans.ToMeanNormalization( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] )
            mtrans.ToNormalization(),
            ])
    )

# emotions = dataset.data.classes
emotions = ['Neutral - NE', 'Happiness - HA', 'Surprise - SU', 'Sadness - SA', 'Anger - AN', 'Disgust - DI', 'Fear - FR', 'Contempt - CO']

# if namedataset == FactoryDataset.bu3dfe:
#     emotions = emotions[:-1]
print(emotions)
print(len(emotions))

import random

def sigmoid(x):
    return 1. / (1 + np.exp(-x))

def norm(x):
    x = x-x.min()
    x = x / x.max()
    return x

def mean_normalization(image, mean, std):
    tensor = image.float()/255.0
    result_tensor = []
    for t, m, s in zip(tensor, mean, std):  
        result_tensor.append(t.sub_(m).div_(s))            
    return torch.stack(result_tensor, 0)

def pad(image, xypad):
    h,w = image.shape
    im_pad = np.zeros( (h+2*xypad,w+2*xypad) )
    im_pad[xypad:xypad+h,xypad:xypad+w] = image
    return im_pad
    
def crop(image, xycrop):
    h,w = image.shape[:2]
    image = image[ xycrop:h-xycrop,xycrop:w-xycrop ]
    return image 
    
imagesize=128
image = cv2.imread('../rec/selfie_happy_dos.png')[:,:,0]
# image = pad(image,50)
# image = crop(image,50)

# sigma=0.1
# image = image/255.0
# noise = np.array([random.gauss(0,sigma) for i in range( image.shape[0]*image.shape[1]  )])
# noise = noise.reshape(image.shape[0],image.shape[1])
# image = (np.clip(image+noise,0,1)*255).astype(np.uint8)

image = np.stack( (image,image,image), axis=2 )
image = cv2.resize( image, (imagesize, imagesize) )

# gamma=0.1
# image[:,:,0] = norm((image[:,:,0]/255)**gamma)*255
# image[:,:,1] = norm((image[:,:,1]/255)**gamma)*255
# image[:,:,2] = norm((image[:,:,2]/255)**gamma)*255

image = torch.from_numpy(image).permute( (2,0,1) ).unsqueeze(0).float()
# image = mean_normalization(image, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
image = image/255

z, y_lab_hat, att, fmap, srf = net( image )


att  = att.data.cpu().numpy().transpose(2,3,1,0)[...,0]
fmap = fmap.data.cpu().numpy().transpose(2,3,1,0)[:,:,0,0]
srf  = srf.data.cpu().numpy().transpose(2,3,1,0)[...,0]
image = image.data.cpu().numpy().transpose(2,3,1,0)[...,0]
y_lab_hat_max = y_lab_hat.argmax()


print(image.shape)
print(y_lab_hat)
print(y_lab_hat_max)
print(emotions[y_lab_hat_max])

plt.figure( figsize=(16,8))
plt.subplot(141)
plt.imshow( norm(image) )
plt.title('image')   
plt.axis('off')
plt.subplot(142)
plt.imshow( (fmap))
plt.title('attention map') 
plt.axis('off' )
plt.subplot(143)
plt.imshow( srf.sum(2) )
plt.title('feature map') 
plt.axis('off' )
plt.subplot(144)
plt.imshow( norm(att) )  
# plt.title('class {}'.format( y_lab_hat_max ) ) 
plt.title('attention feature') 
plt.axis('off')



plt.show()
import scipy.misc
import random

def sigmoid(x):
    return 1. / (1 + np.exp(-x))

def norm(x):
    x = x-x.min()
    x = x / x.max()
    return x

def mean_normalization(image, mean, std):
    tensor = image.float()/255.0
    result_tensor = []
    for t, m, s in zip(tensor, mean, std):  
        result_tensor.append(t.sub_(m).div_(s))            
    return torch.stack(result_tensor, 0)

def pad(image, xypad):
    h,w = image.shape[:2]
    im_pad = np.zeros( (h+2*xypad,w+2*xypad,3) )
    im_pad[xypad:xypad+h,xypad:xypad+w, :] = image
    return im_pad

def crop(image, xycrop):
    h,w = image.shape[:2]
    image = image[ xycrop:h-xycrop,xycrop:w-xycrop , :]
    return image 


def fusion( imx, imy, x=0,y=0, alpha=0.5 ):
    n,m = imy.shape[:2]
    imx[ x:x+n,y:y+m, : ] = alpha*imx[ x:x+n,y:y+m, : ] + (1-alpha)*imy
    return imx

def noise(image, sigma=0.05):
    image = image/255.0
    noise = np.array([random.gauss(0,sigma) for i in range( image.shape[0]*image.shape[1]*3  )])
    noise = noise.reshape(image.shape[0],image.shape[1],3)
    image = (np.clip(image+noise,0,1)*255).astype(np.uint8)
    return image
    
def ligth(image, gamma=0.2):
    image[:,:,0] = norm((image[:,:,0]/255)**gamma)*255
    image[:,:,1] = norm((image[:,:,1]/255)**gamma)*255
    image[:,:,2] = norm((image[:,:,2]/255)**gamma)*255
    return image

class cTrack(object):
    '''track frame
    '''
    
    def __init__(self, net, image_size=128):
        self.imagesize=image_size
        self.net=net
    
    def __call__(self, frame):
        
        #image = frame
        image = frame.mean(axis=2)        
        image = np.stack( (image,image,image), axis=2 )
        
        image = cv2.resize( image, (self.imagesize, self.imagesize) )
        image = torch.from_numpy(image).permute( (2,0,1) ).unsqueeze(0).float()
        #image = mean_normalization(image, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
        image = image/255
        
        zhat, y_lab_hat, att, fmap, srf = self.net( image )
        
        att = att.data.cpu().numpy().transpose(2,3,1,0)[...,0]
        fmap = fmap.data.cpu().numpy().transpose(2,3,1,0)[:,:,0,0]
        srf  = srf.data.cpu().numpy().transpose(2,3,1,0)[...,0]
        y_lab_hat_max = y_lab_hat.argmax()
        
        return att, fmap, srf, zhat, y_lab_hat, y_lab_hat_max

    
class cFrame(object):
    '''frames porcess
    '''        
    def __init__(self, image_size=[640, 640, 3], border=0, offsetx=0, offsety=0):     
        self.imagesize  = image_size
        self.asp = float(image_size[1])/image_size[0]
        self.border = border
        self.offsetx = offsetx
        self.offsety = offsety 
       
        
    def __call__(self, frame):
        '''process frame 
        '''        
        #H, W original image size
        H=frame.shape[0]; W=frame.shape[1];
        
        #image canonization
        if H>W: frame = frame.transpose(1,0,2)
        H=frame.shape[0]; W=frame.shape[1]     
        
        H1 = int(H - self.border)
        W1 = int(H1 * self.asp)
        offsetx=self.offsetx
        offsety=self.offsety
        Wdif = int(np.abs(W - W1) / 2.0)
        Hdif = int(np.abs(H - H1) / 2.0)
        vbox = np.array([[Wdif, Hdif], [W - Wdif, H - Hdif]])

        frame_p = frame[vbox[0, 1]+offsety:vbox[1, 1]+offsety, vbox[0, 0]+offsetx:vbox[1, 0]+offsetx, : ]; #(2, 1, 0)
        aspY = float(self.imagesize[0]) / frame_p.shape[0]
        aspX = float(self.imagesize[1]) / frame_p.shape[1]

        frame_p = scipy.misc.imresize(frame_p, (self.imagesize[0], self.imagesize[1]), interp='bilinear')
        
        return frame_p
    
    
def drawcaption( y, emotions, imsizeout=(200,200) ):    
    
    ne = len(emotions)
    colors = ([150,150,150],[125,125,125],[255,255,255],[255,255,255])
    hbox=40; wbox= 135 + 170
    
    imsize=(hbox*ne,wbox,3)
    imemotions = np.zeros( imsize, dtype=np.uint8 )*255    
    
    ymax = y.argmax()
    
    for i, yi in enumerate(y):

        k  = 1 if y[i]>0.5 else 0
        kh = 1 if ymax==i else 0
    
        bbox = np.array([[0,0],[wbox,0],[wbox,hbox],[0,hbox]]);    
        bbox[:,0] += 0
        bbox[:,1] += 26-26 + (i)*40
        imemotions = cv2.fillConvexPoly(imemotions, bbox, color=colors[kh] )
        
        bbox = np.array([[0,0],[int(wbox*y[i]),0],[int(y[i]*wbox),hbox],[0,hbox]]);    
        bbox[:,0] += 0
        bbox[:,1] += 26-26 + (i)*40
        imemotions = cv2.fillConvexPoly(imemotions, bbox, color=[255,160,122] )
        
        cv2.putText(
                imemotions, 
                #'{}: {:.3f}'.format(emotions[i][:-5],y[i]),
                '{}: {:.2f}%'.format(emotions[i][:-5], y[i]*100 ),
                (2, 26 + (i)*40), 
                color=colors[2+kh], 
                fontFace=cv2.FONT_HERSHEY_SIMPLEX, 
                fontScale=1, 
                thickness=2
                )
        
    #imemotions = imemotions[20:-20,20:-20,:]
    imemotions = cv2.resize( imemotions, imsizeout )
    return imemotions


import cv2 
from IPython.display import clear_output

filename = '../out/videos/sad2.avi'  
cap = cv2.VideoCapture( filename )

print(cap.isOpened())
frame_proc = cFrame( image_size=[128,128,3] ) 
track   = cTrack( net, image_size=128 )

k = 0
iframe=0
totalframe=30
iniframe=0 #0, 1700

bligth=False
mingam, maxgam = 0.5, 3.0 
#gammas = mingam + np.random.rand(totalframe)*( maxgam - mingam )
gammas = np.linspace( mingam, maxgam, num=totalframe+1 )
gammas = gammas[::-1]
#gammas.sort()


bnoise=False
minnoise, maxnoise = 0.01, 0.08
# sigmas = minnoise + np.random.rand(totalframe)*( maxnoise - minnoise )
sigmas = np.linspace( minnoise, maxnoise, num=totalframe+1 )
sigmas = sigmas[::-1]
#sigmas.sort()


# for every frame
while(cap.isOpened()):
    
    # read
    #for i in range(100): 
    #    ret, frame = cap.read()

    ret, frame = cap.read()
    if not ret:
        k+=1
        print('Error video')
        break

    if k%1 != 0 or k < iniframe:
        k+=1
        continue
        
    #print(k)   
    #frame = frame[:-300,850:,:]
    #frame = frame[50:-350,500:1350,:]
    
    #frame = frame[0:500,0:500,:] # happy, sad1
    #frame = frame[0:700,0:700,:]
    #frame = frame[100:700,100:700,:]
    frame = frame[0:320,350:720,:] # sad2(crop 30)
    #frame = frame[50:450,300:720,:] #fear
    
    image = frame_proc( frame )
    image = crop(image,30)
    #image = pad(image, 20)
    #image = ligth(image, gamma = 0.3 )
       
    #noise
    if bnoise:
        image = noise(image, sigma=sigmas[iframe - iniframe])  
        image = np.clip(image, 0, 255 )
    
    #ligth
    if bligth:
        image = ligth(image, gamma = gammas[iframe - iniframe] )
        image = np.clip(image, 0, 255 )
    
    att, fmap, srf, zhat, yhat, yhat_max = track( image )
    yhat = TF.softmax( yhat, dim=1 )[0,:]
    att_map = att.mean(axis=2)
    
    #print(yhat)
    
    
    #create video frame   
    imsize=500
    midsize=250
    layer   = np.zeros( [768, 2024, 3] , dtype=np.uint8 )*255 #1024
    
    caption = drawcaption(yhat, emotions )
    image   = cv2.resize( image, (imsize,imsize) )[:,:,(2,1,0)]
    att     = cv2.resize( norm(att)*255, (imsize,imsize) )
    fmap    = cv2.resize( norm(fmap)*255, (imsize,imsize) )
    srf     = cv2.resize( norm(srf.sum(axis=2))*255, (imsize,imsize) )
    
    #att     = cv2.resize( norm(att)*255, (midsize,midsize) )
    #fmap    = cv2.resize( norm(fmap)*255, (midsize,midsize) )
    #srf     = cv2.resize( norm(srf.sum(axis=2))*255, (midsize,midsize) )
    
    #https://www.learnopencv.com/applycolormap-for-pseudocoloring-in-opencv-c-python/
    fmap = cv2.applyColorMap( fmap.astype(np.uint8) , cv2.COLORMAP_JET)[:,:,(2,1,0)]
    srf = cv2.applyColorMap( srf.astype(np.uint8), cv2.COLORMAP_JET)[:,:,(2,1,0)]
    
    #att     = np.concatenate( (att, att_t), axis=0 )
    #feature = np.concatenate( (fmap, srf ), axis=0 )
    #feature = np.stack( (feature, feature,feature ), axis=2 )    
    #image   = np.concatenate( (image, feature, att), axis=1 )
    
    image   = np.concatenate( (image, fmap, srf, att), axis=1 )
    
    
    #layer
    
    layer = fusion(layer, image, x=10+100, y=10, alpha=0.0 )
    layer = fusion(layer, caption, x=20+100, y=20, alpha=0.2 )
    
    cv2.imwrite('../out/result/{:06d}.png'.format( iframe + 0*(totalframe + 1) ), layer[:,:,(2,1,0)] )
    k+=1; iframe+=1
    
    # show
    ishow=False
    if ishow:
        plt.figure( figsize=(25,8) )
        plt.imshow( layer )
        plt.axis('off')
        plt.show()
        clear_output(wait=True) 
    
    if iframe%100 == 0:
        print(k, iframe)        
    
    if k > iniframe+totalframe:
        break

    
        
        
cap.release()
print('DONE!!!')<jupyter_output>True
DONE!!!
{'fraction_non_alphanumeric': 0.17466881078183627, 'fraction_numerical': 0.05367945478214258, 'mean_word_length': 2.662366797532249, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '227733', 'license': 'MIT', 'n_tokens_mistral': 5596, 'n_tokens_neox': 5147, 'n_words': 912, 'path': 'books/track_gmm.ipynb', 'repo_name': 'daaiwusheng/ferattention', 'size': '13059'}