<jupyter_start><jupyter_text>Codice della repository [rcmalli](https://github.com/rcmalli/keras-vggface) per permettere il funzionamento di keras con la GPU<jupyter_code>import numpy as np
from keras import backend as K
from keras.utils.data_utils import get_file

V1_LABELS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_labels_v1.npy'
V2_LABELS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_labels_v2.npy'

VGG16_WEIGHTS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_vgg16.h5'
VGG16_WEIGHTS_PATH_NO_TOP = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_notop_vgg16.h5'


RESNET50_WEIGHTS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_resnet50.h5'
RESNET50_WEIGHTS_PATH_NO_TOP = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_notop_resnet50.h5'

SENET50_WEIGHTS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_senet50.h5'
SENET50_WEIGHTS_PATH_NO_TOP = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_notop_senet50.h5'


VGGFACE_DIR = 'models/vggface'


def preprocess_input(x, data_format=None, version=1):
    x_temp = np.copy(x)
    if data_format is None:
        data_format = K.image_data_format()
    assert data_format in {'channels_last', 'channels_first'}

    if version == 1:
        if data_format == 'channels_first':
            x_temp = x_temp[:, ::-1, ...]
            x_temp[:, 0, :, :] -= 93.5940
            x_temp[:, 1, :, :] -= 104.7624
            x_temp[:, 2, :, :] -= 129.1863
        else:
            x_temp = x_temp[..., ::-1]
            x_temp[..., 0] -= 93.5940
            x_temp[..., 1] -= 104.7624
            x_temp[..., 2] -= 129.1863

    elif version == 2:
        if data_format == 'channels_first':
            x_temp = x_temp[:, ::-1, ...]
            x_temp[:, 0, :, :] -= 91.4953
            x_temp[:, 1, :, :] -= 103.8827
            x_temp[:, 2, :, :] -= 131.0912
        else:
            x_temp = x_temp[..., ::-1]
            x_temp[..., 0] -= 91.4953
            x_temp[..., 1] -= 103.8827
            x_temp[..., 2] -= 131.0912
    else:
        raise NotImplementedError

    return x_temp


def decode_predictions(preds, top=5):
    LABELS = None
    if len(preds.shape) == 2:
        if preds.shape[1] == 2622:
            fpath = get_file('rcmalli_vggface_labels_v1.npy',
                             V1_LABELS_PATH,
                             cache_subdir=VGGFACE_DIR)
            LABELS = np.load(fpath)
        elif preds.shape[1] == 8631:
            fpath = get_file('rcmalli_vggface_labels_v2.npy',
                             V2_LABELS_PATH,
                             cache_subdir=VGGFACE_DIR)
            LABELS = np.load(fpath)
        else:
            raise ValueError('`decode_predictions` expects '
                             'a batch of predictions '
                             '(i.e. a 2D array of shape (samples, 2622)) for V1 or '
                             '(samples, 8631) for V2.'
                             'Found array with shape: ' + str(preds.shape))
    else:
        raise ValueError('`decode_predictions` expects '
                         'a batch of predictions '
                         '(i.e. a 2D array of shape (samples, 2622)) for V1 or '
                         '(samples, 8631) for V2.'
                         'Found array with shape: ' + str(preds.shape))
    results = []
    for pred in preds:
        top_indices = pred.argsort()[-top:][::-1]
        result = [[str(LABELS[i].encode('utf8')), pred[i]] for i in top_indices]
        result.sort(key=lambda x: x[1], reverse=True)
        results.append(result)
    return results<jupyter_output><empty_output><jupyter_text>Data Augmentation CV2<jupyter_code>numberTransformation = 12
permuteTransf = True

def fill(img, h, w):
    img = cv2.resize(img, (h, w), cv2.INTER_CUBIC)
    return img

def horizontal_shift(img, ratio=0.0):
    if ratio > 1 or ratio < 0:
        print('Value should be less than 1 and greater than 0')
        return img
    ratio = random.uniform(-ratio, ratio)
    h, w = img.shape[:2]
    to_shift = w*ratio
    if ratio > 0:
        img = img[:, :int(w-to_shift), :]
    if ratio < 0:
        img = img[:, int(-1*to_shift):, :]
    img = fill(img, h, w)
    return img

def vertical_shift(img, ratio=0.0):
    if ratio > 1 or ratio < 0:
        print('Value should be less than 1 and greater than 0')
        return img
    ratio = random.uniform(-ratio, ratio)
    h, w = img.shape[:2]
    to_shift = h*ratio
    if ratio > 0:
        img = img[:int(h-to_shift), :, :]
    if ratio < 0:
        img = img[int(-1*to_shift):, :, :]
    img = fill(img, h, w)
    return img

def brightness(img, low, high):
    value = random.uniform(low, high)
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    hsv = np.array(hsv, dtype = np.float64)
    hsv[:,:,1] = hsv[:,:,1]*value
    hsv[:,:,1][hsv[:,:,1]>255]  = 255
    hsv[:,:,2] = hsv[:,:,2]*value 
    hsv[:,:,2][hsv[:,:,2]>255]  = 255
    hsv = np.array(hsv, dtype = np.uint8)
    img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
    return img

def zoom(img, value):
    if value > 1 or value < 0:
        print('Value for zoom should be less than 1 and greater than 0')
        return img
    value = random.uniform(value, 1)
    h, w = img.shape[:2]
    h_taken = int(value*h)
    w_taken = int(value*w)
    h_start = random.randint(0, h-h_taken)
    w_start = random.randint(0, w-w_taken)
    img = img[h_start:h_start+h_taken, w_start:w_start+w_taken, :]
    img = fill(img, h, w)
    return img

def channel_shift(img, value):
    value = int(random.uniform(-value, value))
    img = img + value
    img[:,:,:][img[:,:,:]>255]  = 255
    img[:,:,:][img[:,:,:]<0]  = 0
    img = img.astype(np.uint8)
    return img

def horizontal_flip(img, flag):
    if flag:
        return cv2.flip(img, 1)
    else:
        return img

def vertical_flip(img, flag):
    if flag:
        return cv2.flip(img, 0)
    else:
        return img

def rotation(img, angle):
    angle = int(random.uniform(-angle, angle))
    h, w = img.shape[:2]
    M = cv2.getRotationMatrix2D((int(w/2), int(h/2)), angle, 1)
    img = cv2.warpAffine(img, M, (w, h))
    return img

def colorjitter(img, cj_type=1):

    if cj_type == 1:
        # value = random.randint(-50, 50)
        value = np.random.choice(np.array([-50, -40, -30, 30, 40, 50]))
        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        h, s, v = cv2.split(hsv)
        if value >= 0:
            lim = 255 - value
            v[v > lim] = 255
            v[v <= lim] += value
        else:
            lim = np.absolute(value)
            v[v < lim] = 0
            v[v >= lim] -= np.absolute(value)

        final_hsv = cv2.merge((h, s, v))
        img = cv2.cvtColor(final_hsv, cv2.COLOR_HSV2BGR)
        return img
    
    elif cj_type == 2:
        # value = random.randint(-50, 50)
        value = np.random.choice(np.array([-50, -40, -30, 30, 40, 50]))
        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        h, s, v = cv2.split(hsv)
        if value >= 0:
            lim = 255 - value
            s[s > lim] = 255
            s[s <= lim] += value
        else:
            lim = np.absolute(value)
            s[s < lim] = 0
            s[s >= lim] -= np.absolute(value)

        final_hsv = cv2.merge((h, s, v))
        img = cv2.cvtColor(final_hsv, cv2.COLOR_HSV2BGR)
        return img
    
    elif cj_type == 3:
        brightness = 10
        contrast = random.randint(40, 100)
        dummy = np.int16(img)
        dummy = dummy * (contrast/127+1) - contrast + brightness
        dummy = np.clip(dummy, 0, 255)
        img = np.uint8(dummy)
        return img

def noisy(img, noise_type=1):

    if noise_type == 1:
        image=img.copy() 
        mean=0
        st=0.7
        gauss = np.random.normal(mean,st,image.shape)
        gauss = gauss.astype('uint8')
        image = cv2.add(image,gauss)
        return image
    
    elif noise_type == 2:
        image=img.copy() 
        prob = 0.05
        if len(image.shape) == 2:
            black = 0
            white = 255            
        else:
            colorspace = image.shape[2]
            if colorspace == 3:  # RGB
                black = np.array([0, 0, 0], dtype='uint8')
                white = np.array([255, 255, 255], dtype='uint8')
            else:  # RGBA
                black = np.array([0, 0, 0, 255], dtype='uint8')
                white = np.array([255, 255, 255, 255], dtype='uint8')
        probs = np.random.random(image.shape[:2])
        image[probs < (prob / 2)] = black
        image[probs > 1 - (prob / 2)] = white
        return image

def filters(img, f_type = 1):
    if f_type == 1:
        image=img.copy()
        fsize = 9
        return cv2.blur(image,(fsize,fsize))
    
    elif f_type == 2:
        image=img.copy()
        fsize = 9
        return cv2.GaussianBlur(image, (fsize, fsize), 0)
    
    elif f_type == 3:
        image=img.copy()
        fsize = 9
        return cv2.medianBlur(image, fsize)

#la funzione effettua una singola trasformazione se permuteTransf = False
#o una permutazioni di trasformazioni se permuteTransf = True
def modifyImage(img):
  approx = 2

  if permuteTransf:
    maxNum = numberTransformation
  else:
    maxNum = 1

  numberOfCicles = random.randint(0, maxNum)

  finalImage = img

  for i in range(numberOfCicles):
    transf = random.randint(1, numberTransformation)
    if transf == 1:
      finalImage = horizontal_shift(finalImage, round(random.uniform(0.1,0.9), approx))
    elif transf == 2:
      finalImage = vertical_shift(finalImage, round(random.uniform(0.1,0.9), approx))
    elif transf == 3:
      min = round(random.uniform(0.1, 0.9), approx)
      max = round(random.uniform(min, 0.9), approx)
      finalImage = brightness(finalImage, min, max)
    elif transf == 4:
      finalImage = zoom(finalImage, round(random.uniform(0.1,0.9), approx))
    elif transf == 5:
      finalImage = channel_shift(finalImage, random.randint(0, 255))
    elif transf == 6:
      finalImage = horizontal_flip(finalImage, bool(random.getrandbits(1)))
    elif transf == 7:
      finalImage = vertical_flip(finalImage, bool(random.getrandbits(1)))
    elif transf == 8:
      finalImage = rotation(finalImage, random.randint(0, 360))
    elif transf == 9:
      finalImage = colorjitter(finalImage, random.randint(1, 3))
    elif transf == 10:
      finalImage = noisy(finalImage, random.randint(1, 2))
    elif transf == 11:
      finalImage = filters(finalImage, random.randint(1, 2))

  return finalImage

#analoga alla funzione modifyImage con la differenza che applica le stesse trasformazioni 
#a due immagini diverse
def modifyMultImage(img1, img2):
  approx = 2

  if permuteTransf:
    maxNum = numberTransformation
  else:
    maxNum = 1
    
  numberOfCicles = random.randint(0, maxNum)

  finalImage1 = img1
  finalImage2 = img2

  for i in range(numberOfCicles):
    transf = random.randint(1, numberTransformation)
    if transf == 1:
      tmp = round(random.uniform(0.1,0.9), approx)
      finalImage1 = horizontal_shift(finalImage1, tmp)
      finalImage2 = horizontal_shift(finalImage2, tmp)
    elif transf == 2:
      tmp =  round(random.uniform(0.1,0.9), approx)
      finalImage1 = vertical_shift(finalImage1, tmp)
      finalImage2 = vertical_shift(finalImage2, tmp)
    elif transf == 3:
      min = round(random.uniform(0.1, 0.9), approx)
      max = round(random.uniform(min, 0.9), approx)
      finalImage1 = brightness(finalImage1, min, max)
      finalImage2 = brightness(finalImage2, min, max)
    elif transf == 4:
      tmp = round(random.uniform(0.1,0.9), approx)
      finalImage1 = zoom(finalImage1, tmp)
      finalImage2 = zoom(finalImage2, tmp)
    elif transf == 5:
      tmp = random.randint(0, 255)
      finalImage1 = channel_shift(finalImage1, tmp)
      finalImage2 = channel_shift(finalImage2, tmp)
    elif transf == 6:
      tmp = bool(random.getrandbits(1))
      finalImage1 = horizontal_flip(finalImage1, tmp)
      finalImage2 = horizontal_flip(finalImage2, tmp)
    elif transf == 7:
      tmp = bool(random.getrandbits(1))
      finalImage1 = vertical_flip(finalImage1, tmp)
      finalImage2 = vertical_flip(finalImage2, tmp)
    elif transf == 8:
      tmp = random.randint(0, 360)
      finalImage1 = rotation(finalImage2, tmp)
      finalImage1 = rotation(finalImage2, tmp)
    elif transf == 9:
      tmp = random.randint(1, 3)
      finalImage1 = colorjitter(finalImage2, tmp)
      finalImage2 = colorjitter(finalImage2, tmp)
    elif transf == 10:
      tmp = random.randint(1, 2)
      finalImage1 = noisy(finalImage1, tmp)
      finalImage2 = noisy(finalImage2, tmp)
    elif transf == 11:
      tmp = random.randint(1, 2)
      finalImage1 = filters(finalImage1, tmp)
      finalImage2 = filters(finalImage2, tmp)

  return finalImage1, finalImage2 
imageAugmentation = True
#se applicare o meno le stesse trasformazioni a due immagini diverse della stessa coppia
useSameTransformation = False
#se invertire o meno le persone di una coppia
invertRel = True
imgWidth = 224
imgHeight = 224
epochs = 30
stepsPerEpoch = 200
verbose = 1
batchSize = 16
validationSteps = 100

#path dei file math
fdPairsPath = "/content/drive/MyDrive/ProgettoFVAB/ProgettoFinale/input/meta_data/fd_pairs.mat"
fsPairsPath = "/content/drive/MyDrive/ProgettoFVAB/ProgettoFinale/input/meta_data/fs_pairs.mat"
mdPairsPath = "/content/drive/MyDrive/ProgettoFVAB/ProgettoFinale/input/meta_data/md_pairs.mat"
msPairsPath = "/content/drive/MyDrive/ProgettoFVAB/ProgettoFinale/input/meta_data/ms_pairs.mat"
fdDir = "father-dau/"
fsDir = "father-son/"
mdDir = "mother-dau/"
msDir = "mother-son/"

#path modelli
trainPath = "/content/drive/MyDrive/ProgettoFVAB/ProgettoFinale/input/trainW2/"
modelPath = "/content/drive/MyDrive/ProgettoFVAB/ProgettoFinale/Modelli e risultati/vgg_face77Acc.h5"
#modelPath = "/content/drive/MyDrive/ProgettoFVAB/baselineGGG.h5"



X = []
Y = []
labels = []

testX = []
testY = []
testLabels = []

#la funzione dato il valore del kin in input e il tipo di relazione
#restituisce un array contenente il kin per l'output a 5 classi
def prepareKin(dirName, kin):
  if dirName == fdDir:
    if kin == 1:
      return (kin, 0, 0, 0, 0)
    else:
      return (0, 0, 0, 0, 1)
  elif dirName == fsDir:
    if kin == 1:
      return (0, kin, 0, 0, 0)
    else:
      return (0, 0, 0, 0, 1)
  elif dirName == mdDir:
    if kin == 1:
      return (0, 0, kin, 0, 0)
    else:
      return (0, 0, 0, 0, 1)
  elif dirName == msDir:
    if kin == 1:
      return (0, 0, 0, kin, 0)
    else:
      return (0, 0, 0, 0, 1)

#funzione analoga a prepareKin utilizzata per debugging di codice
def prepareKin2(dirName, kin):
  if dirName == fdDir:
    if kin == 1:
      return [kin, 0, 0, 0, 0]
    else:
      return [0, 0, 0, 0, 1]
  elif dirName == fsDir:
    if kin == 1:
      return [0, kin, 0, 0, 0]
    else:
      return [0, 0, 0, 0, 1]
  elif dirName == mdDir:
    if kin == 1:
      return [0, 0, kin, 0, 0]
    else:
      return [0, 0, 0, 0, 1]
  elif dirName == msDir:
    if kin == 1:
      return [0, 0, 0, kin, 0]
    else:
      return [0, 0, 0, 0, 1]

#la funzione riceve in input la struttura dati x da cui 
#preleva il path delle due immagini da leggere.
#in seguito se useSameTransformation = True applica la stessa trasformazione ad entrambe le immagini
#altrimenti applica trasformazioni diverse
def prepareImage(x, kin, dirName, imgAUG = False):
  image1 = (x[2])[0]
  image2 = (x[3])[0]
  image1Path = trainPath + dirName + image1
  image2Path = trainPath + dirName + image2
  
  if useSameTransformation:
    img1, img2 = read_mul_img(image1Path, image2Path, imgAUG)
  else:
    img1 = read_img(image1Path, imgAUG)
    img2 = read_img(image2Path, imgAUG)

  return img1, img2

#funzione utilizzata durante le diverse prove di training che crea una batch perfetta:
#4 immagini padre-figlia
#4 immagini padre-figlio
#4 immagini madre-figlia
#4 immagini madre-figlio
#4 immagini non-kin, una per ogni relazione
def genFixed(X, Y, labels, validation = False, imgAUG = False):

  while True:
    
    temp1 = []
    temp2 = []
    tempLabel = []

    if validation:
      range1 = 300 
      range2 = 399
    else:
      range1 = 0
      range2 = 299

    matFile1 = fdPairsPath
    matFile2 = fsPairsPath
    matFile3 = mdPairsPath
    matFile4 = msPairsPath

    #FILE1
    countKin = 0
    countNonKin = 0
    while True:
      data = loadmat(matFile1)
      data = data['pairs']
      n = random.randint(range1, range2)
      x = data[n]
      kin = ((x[1])[0])[0]
        

      if kin == 0 and countNonKin != 1:
        img1, img2 = prepareImage(x, kin, fdDir, imgAUG)
        kin = prepareKin(fdDir, kin)
        temp1.append(img1)
        temp2.append(img2)
        tempLabel.append(kin)
        countNonKin = countNonKin + 1
      elif kin == 1 and countKin != 4:
        img1, img2 = prepareImage(x, kin, fdDir, imgAUG)
        kin = prepareKin(fdDir, kin)
        temp1.append(img1)
        temp2.append(img2)
        tempLabel.append(kin)
        countKin = countKin +1

      if countKin == 4 and countNonKin == 1:
        break

      #FILE 2 
    countKin = 0
    countNonKin = 0
    while True:
      data = loadmat(matFile2)
      data = data['pairs']
      n = random.randint(range1, range2)
      x = data[n]
      kin = ((x[1])[0])[0]
        
      if kin == 0 and countNonKin != 1:
        img1, img2 = prepareImage(x, kin, fsDir, imgAUG)
        kin = prepareKin(fsDir, kin)
        temp1.append(img1)
        temp2.append(img2)
        tempLabel.append(kin)
        countNonKin = countNonKin + 1
      elif kin == 1 and countKin != 4:
        img1, img2 = prepareImage(x, kin, fsDir, imgAUG)
        kin = prepareKin(fsDir, kin)
        temp1.append(img1)
        temp2.append(img2)
        tempLabel.append(kin)
        countKin = countKin +1

      if countKin == 4 and countNonKin == 1:
        break

      #FILE 3
    countKin = 0
    countNonKin = 0
    while True:
      data = loadmat(matFile3)
      data = data['pairs']
      n = random.randint(range1, range2)
      x = data[n]
      kin = ((x[1])[0])[0]
        
      if kin == 0 and countNonKin != 1:
        img1, img2 = prepareImage(x, kin, mdDir, imgAUG)
        kin = prepareKin(mdDir, kin)
        temp1.append(img1)
        temp2.append(img2)
        tempLabel.append(kin)
        countNonKin = countNonKin + 1
      elif kin == 1 and countKin != 4:
        img1, img2 = prepareImage(x, kin, mdDir, imgAUG)
        kin = prepareKin(mdDir, kin)
        temp1.append(img1)
        temp2.append(img2)
        tempLabel.append(kin)
        countKin = countKin +1

      if countKin == 4 and countNonKin == 1:
        break
    
    #FILE4
    countKin = 0
    countNonKin = 0
    while True:
      data = loadmat(matFile4)
      data = data['pairs']
      n = random.randint(range1, range2)
      x = data[n]
      kin = ((x[1])[0])[0]
        
      if kin == 0 and countNonKin != 1:
        img1, img2 = prepareImage(x, kin, msDir, imgAUG)
        kin = prepareKin(msDir, kin)
        temp1.append(img1)
        temp2.append(img2)
        tempLabel.append(kin)
        countNonKin = countNonKin + 1
      elif kin == 1 and countKin != 4:
        img1, img2 = prepareImage(x, kin, msDir, imgAUG)
        kin = prepareKin(msDir, kin)
        temp1.append(img1)
        temp2.append(img2)
        tempLabel.append(kin)
        countKin = countKin +1

      if countKin == 4 and countNonKin == 1:
        break

    
    temp1 = np.array(temp1) 
    temp2 = np.array(temp2) 
    tempLabel = np.array(tempLabel)
    yield [temp1, temp2], tempLabel

#la funzione prepara la bath per l'addestramento
#se invertRel = True la coppia viene invertita.
def gen(X, Y, labels, validation = False, imgAUG = False):
  
  #le prime 300 coppie sono per il training
  #dalla 301 alla 400 sono per l'addestramento
  if validation:
    range1 = 300 
    range2 = 399
  else:
    range1 = 0
    range2 = 299

  while True:
    #sceglie una delle 4 relazioni e prepara i parametri
    numFile = random.randint(0,3)
    matFilePath = ''
    dirName = ''

    #sceglie il file math da utilizzare
    if numFile == 0:
      matFilePath = fdPairsPath
      dirName = fdDir
    elif numFile == 1:
      matFilePath = fsPairsPath
      dirName = fsDir
    elif numFile == 2:
      matFilePath = mdPairsPath
      dirName = mdDir
    elif numFile == 3:
      matFilePath = msPairsPath
      dirName = msDir

    data = loadmat(matFilePath)
    data = data['pairs']

    temp1 = []
    temp2 = []
    tempLabel = []

    #limita il numero di immagini che non hanno una relazione
    negativeLimit = batchSize // 8
    positiveLimit = batchSize - negativeLimit
    negativeCount = 0
    positiveCount = 0
    numFile = 0

    while True:

      if numFile == 0:
        matFilePath = fdPairsPath
        dirName = fdDir
      elif numFile == 1:
        matFilePath = fsPairsPath
        dirName = fsDir
      elif numFile == 2:
        matFilePath = mdPairsPath
        dirName = mdDir
      elif numFile == 3:
        matFilePath = msPairsPath
        dirName = msDir

      data = loadmat(matFilePath)
      data = data['pairs']

      n = random.randint(range1, range2)
      x = data[n]
      kin = ((x[1])[0])[0]

      #verifica se l'immagine è NON-KIN e non è stato raggiunto il limite
      if kin == 0 and negativeCount != negativeLimit:
        img1, img2 = prepareImage(x, kin, dirName, imgAUG)
        kin = prepareKin(dirName, kin)

        if invertRel:
          if random.randint(0,1) == 0:
            temp2.append(img1)
            temp1.append(img2)
          else:
            temp1.append(img1)
            temp2.append(img2)
        else:
          temp1.append(img1)
          temp2.append(img2)

        tempLabel.append(kin)
        negativeCount = negativeCount + 1

         #verifica se l'immagine HA RELAZIONE e non è stato raggiunto il limite
      elif kin == 1 and positiveCount != positiveLimit:
        img1, img2 = prepareImage(x, kin, dirName, imgAUG)
        kin = prepareKin(dirName, kin)

        if invertRel:
          if random.randint(0,1) == 0:
            temp2.append(img1)
            temp1.append(img2)
          else:
            temp1.append(img1)
            temp2.append(img2)
        else:
          temp1.append(img1)
          temp2.append(img2)


        tempLabel.append(kin)
        positiveCount = positiveCount +1

      if positiveCount == positiveLimit and negativeCount == negativeLimit:
        break
      
      numFile = (numFile + 1) % 4
    
    temp1 = np.array(temp1) 
    temp2 = np.array(temp2) 
    tempLabel = np.array(tempLabel)
    yield [temp1, temp2], tempLabel

#analoga a read_img ma legge due immagini contemporaneamente
def read_mul_img(path1, path2, imgAUG = False):
    dim = (imgWidth, imgHeight)
    img1 = cv2.imread(path1)
    img2 = cv2.imread(path2)
    img1 = cv2.resize(img1, dim)
    img2 = cv2.resize(img2, dim)

    if imgAUG:
      img1, img2 = modifyMultImage(img1, img2)
      
    img1 = np.array(img1).astype(np.float)
    img2 = np.array(img2).astype(np.float)
    img1 = preprocess_input(img1, version=2)
    img2 = preprocess_input(img2, version=2)
    return img1, img2

#legge le immagini e le ridimensiona
#se imgAUG = True effettua image augumentation
def read_img(path, imgAUG = False):
    dim = (imgWidth, imgHeight)
    img = cv2.imread(path)
    
    img = cv2.resize(img, dim)

    if imgAUG:
      img = modifyImage(img)
      
    img = np.array(img).astype(np.float)
    return preprocess_input(img, version=2)

#analoga a read_img ma utilizzata per preparare le immagini per il testing del modello
def read_imgTesting(path, imgAUG = False):
    dim = (imgWidth, imgHeight)
    img = cv2.imread(path)
    
    img = cv2.resize(img, dim)
    img = img.reshape(1,224,224,3)

    if imgAUG:
      img = modifyImage(img)
      
    img = np.array(img).astype(np.float)
    return preprocess_input(img, version=2)

#prepara le immagini per il testing inserendole tutte
#in un unico array
def prepareTestData(pairsPath, dirName):
  data = loadmat(pairsPath)

  #array([[array([[1]], dtype=uint8), array([[1]], dtype=uint8),
       # array(['fd_240_1.jpg'], dtype='<U12'),
        #array(['fd_240_2.jpg'], dtype='<U12')],...
  data = data['pairs']
  # x è del tipo:
  # array([array([[1]], dtype=uint8), array([[1]], dtype=uint8),
  # array(['fd_240_1.jpg'], dtype='<U12'),
  # array(['fd_240_2.jpg'], dtype='<U12')], dtype=object)
  for x in data:  
    dir = ((x[0])[0])[0]
    tempKin = ((x[1])[0])[0]
    image1 = (x[2])[0]
    image2 = (x[3])[0]

    kin = prepareKin(dirName, tempKin)

    #test positivi e negativi
    if dir == 5:
      image1Path = trainPath + dirName + image1
      image2Path = trainPath + dirName + image2
      img1 = read_imgTesting(image1Path)
      img2 = read_imgTesting(image2Path)
      testX.append(img1)
      testY.append(img2)
      testLabels.append(kin)

#la funzione prepara i dati per il training inserendo tutte 
#le immagini all'interno di un'unica batch
def prepareTrainData(pairsPath, dirName):
  data = loadmat(pairsPath)

  #array([[array([[1]], dtype=uint8), array([[1]], dtype=uint8),
       # array(['fd_240_1.jpg'], dtype='<U12'),
        #array(['fd_240_2.jpg'], dtype='<U12')],...
  data = data['pairs']
  # x è del tipo:
  # array([array([[1]], dtype=uint8), array([[1]], dtype=uint8),
  # array(['fd_240_1.jpg'], dtype='<U12'),
  # array(['fd_240_2.jpg'], dtype='<U12')], dtype=object)


  for x in data:  
    dir = ((x[0])[0])[0]
    tempKin = ((x[1])[0])[0]
    image1 = (x[2])[0]
    image2 = (x[3])[0]

    kin = prepareKin(dirName, tempKin)

    if dir != 5 and dir != 4:
      image1Path = trainPath + dirName + image1
      image2Path = trainPath + dirName + image2
      img1 = read_img(image1Path)
      img2 = read_img(image2Path)

      #verifica se invertire le relazioni
      if invertRel:
        if random.randint(0,1) == 0:
          Y.append(img1)
          X.append(img2)
        else:
          X.append(img1)
          Y.append(img2)
      else:
        X.append(img1)
        Y.append(img2)

      labels.append(kin)




prepareTrainData(fdPairsPath, fdDir)
prepareTrainData(fsPairsPath, fsDir)
prepareTrainData(mdPairsPath, mdDir)
prepareTrainData(msPairsPath, msDir)

X = np.array(X) 
Y = np.array(Y) 
labels = np.array(labels)<jupyter_output><empty_output><jupyter_text>Funzioni per generare un modello da zero e stampare i grafici per l'accuratezza e la loss<jupyter_code>def createNewModel():
  input_1 = Input(shape=(64, 64, 3))
  input_2 = Input(shape=(64, 64, 3))


  base_model = ResNet50(weights='imagenet', include_top=False)

  for x in base_model.layers[:-3]:
        x.trainable = True

  x1 = base_model(input_1)
  x2 = base_model(input_2)

  x1 = Concatenate(axis=-1)([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)])
  x2 = Concatenate(axis=-1)([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)])

  x3 = Subtract()([x1, x2])
  x3 = Multiply()([x3, x3])

  x = Multiply()([x1, x2])

  x = Concatenate(axis=-1)([x, x3])
  x = Dense(100, activation="relu")(x)
  x = Dropout(0.01)(x)
  out = Dense(5, activation="sigmoid")(x)

  model = Model([input_1, input_2], out)
  model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam(0.0001))

  model.summary()

  return model

def plotAccuracy(history):
  plt.plot(history.history['accuracy'])
  plt.plot(history.history['val_accuracy'])
  plt.title('model accuracy')
  plt.ylabel('accuracy')
  plt.xlabel('epoch')
  plt.legend(['train', 'val'], loc='upper left')
  plt.show()

def plotLoss(history):
  plt.plot(history.history['loss'])
  plt.plot(history.history['val_loss'])
  plt.title('model loss')
  plt.ylabel('loss')
  plt.xlabel('epoch')
  plt.legend(['train', 'val'], loc='upper left')
  plt.show()<jupyter_output><empty_output><jupyter_text>Nella cella viene caricato il modello, viene modificato aggiungendo strati finali, vengono aggiunte funzioni di call back e viene addestrato<jupyter_code>oldModel = load_model(modelPath)
oldModel.summary()

#---------------------

#dropOut = Dropout(0.4)(oldModel.layers[-4].output)
#x = Dense(100, activation="relu", name="dense_relu")(dropOut)
#preds = Dense(5,activation='softmax', name="dense_output")(x)

#---------------------
#dropOut = Dropout(0.2)(oldModel.layers[-4].output)
#x = Dense(100, activation="relu", kernel_regularizer=regularizers.l2(0.003), name="dense_relu")(dropOut)
#preds = Dense(5,activation='softmax', name="dense_output")(x)
#---------------------

#-------------------------
#x = Dense(100, activation="relu",name="dense_relu")(oldModel.layers[-4].output)
#preds = Dense(5,activation='softmax', name="dense_output")(x)
#-------------------------

#--------------------
#x = Dense(100, activation="relu", kernel_regularizer=regularizers.l2(0.003), name="dense_relu")(oldModel.layers[-4].output)
#dropOut = Dropout(0.2)(x)
#preds = Dense(5,activation='softmax', name="dense_output")(dropOut)
#--------------------
x = Dense(1024, activation="relu", kernel_regularizer=regularizers.l2(0.003))(oldModel.layers[-4].output)
x = Dropout(0.3)(x)
x = Dense(512, activation="relu", kernel_regularizer=regularizers.l2(0.003))(x)
x = Dropout(0.3)(x)
x = Dense(256, activation="relu", kernel_regularizer=regularizers.l2(0.003))(x)
x = Dropout(0.3)(x)
preds = Dense(5,activation='softmax', name="dense_output")(x)

model = Model(inputs=oldModel.input,outputs=preds)


#for layer in oldModel.layers:
  #layer.trainable = False

#for layer in model.layers[0:]:
 #layer.trainable = False


#for layer in model.layers[13:]:
  #layer.trainable=True

file_path = "/content/drive/MyDrive/Colab Notebooks/PIPPOFreezato.h5"

#funzioni di callback per il training del modello
checkpoint = ModelCheckpoint(file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')

#es_callback = EarlyStopping(monitor='val_loss', patience = 200)

reduce_on_plateau = ReduceLROnPlateau(monitor="val_accuracy", mode="max", factor=0.1, patience = 15, verbose=1)

callbacks_list = [checkpoint, reduce_on_plateau]

model.compile(optimizer=Adam(0.00001),loss='categorical_crossentropy',metrics=['accuracy'])

print("")
print("---------------------------------------------------------------------------------")
print("")
print("NUOVO MODELLO")
print("")
print("---------------------------------------------------------------------------------")
print("")

model.summary()

plot_model(model, to_file='/content/drive/MyDrive/ProgettoFVAB/model_plot.png', show_shapes=True, show_layer_names=True)

history = model.fit(x = [X, Y], y = labels, validation_data = gen(X, Y, labels, validation = True, imgAUG = False), callbacks = callbacks_list, epochs = epochs, verbose = verbose, 
                              steps_per_epoch = stepsPerEpoch, validation_steps = validationSteps,
                              workers=1, use_multiprocessing=False, shuffle = True)

#history = model.fit_generator(gen(X, Y, labels,  imgAUG = imageAugmentation),  
#                              validation_data=gen(X, Y, labels, validation = True, imgAUG = False), 
#                              callbacks = callbacks_list, epochs = epochs, verbose = verbose, 
#                              steps_per_epoch = stepsPerEpoch, validation_steps = validationSteps,
#                              workers=1, use_multiprocessing=False)


plotAccuracy(history)
plotLoss(history)
model = load_model("/content/drive/MyDrive/ProgettoFVAB/ProgettoFinale/vgg_faceTransferLearningAUGFREEZE777Bohroco.h5")

prepareTestData(fdPairsPath, fdDir)
prepareTestData(fsPairsPath, fsDir)
prepareTestData(mdPairsPath, mdDir)
prepareTestData(msPairsPath, msDir)


i = 0
predictions = []
#prende una coppia di immagini dall'array e la da in input
#al modello per effettuare le prediction
for x in testX:
  x = model.predict([x, testY[i]], verbose = 1)
  predictions.append(x)
  i = i + 1


fdArray = []
fsArray = []
mdArray = []
msArray = []
nkArray = []
fd = 0
fs = 0
md = 0
ms = 0
nk = 0
fd2 = 0
fs2 = 0
md2 = 0
ms2 = 0
nk2 = 0
bool1 = True
bool2 = True
bool3 = True

count1 = 0
i = 0
#verifica se la prediction è corretta.
#in tal caso incrementa di un'unità il count.
#effettua anche le operazioni necessarie per la costruzione della
#matrice di confusione a 5 classi 
for x in predictions:
  label = testLabels[i]
  print(label)
  indPrediction = np.argmax(x)
  print(indPrediction)

  #relazione FD
  if i < 99:
    #coppie con relazione
    if label[4] == 0:
      if 0 == indPrediction:
        count1 = count1 + 1
        fd = fd +1
      elif 1 == indPrediction:
        fs = fs + 1
      elif 2 == indPrediction:
        md = md + 1
      elif 3 == indPrediction:
        ms = ms + 1
      elif 4 == indPrediction:
        nk = nk + 1
      #coppie NON-KIN
    else:
      if 4 == indPrediction:
        count1 = count1 + 1
        nk2 = nk2 + 1
      elif 0 == indPrediction:
        fd2 = fd2 + 1
      elif 1 == indPrediction:
        fs2 = fs2 + 1
      elif 2 == indPrediction:
        md2 = md2 + 1
      elif 3 == indPrediction:
        ms2 = ms2 + 1
  #RELAZIONE FS
  elif i < 199:
    #aggiunta previsione agli array per la matrice di confusione
    if bool1:
      fdArray.append(fd)
      fdArray.append(fs)
      fdArray.append(md)
      fdArray.append(ms)
      fdArray.append(nk)
      fd = 0
      fs = 0
      md = 0
      ms = 0
      nk = 0
      bool1 = False
      #verifica relazione
    if label[4] == 0:
      if 1 == indPrediction:
        count1 = count1 + 1
        fs = fs + 1
      elif 0 == indPrediction:
        fd = fd + 1
      elif 2 == indPrediction:
        md = md + 1
      elif 3 == indPrediction:
        ms = ms + 1
      elif 4 == indPrediction:
        nk = nk + 1
        #immagini NON-KIN
    else:
      if 4 == indPrediction:
        count1 = count1 + 1
        nk2 = nk2 + 1
      elif 0 == indPrediction:
        fd2 = fd2 + 1
      elif 1 == indPrediction:
        fs2 = fs2 + 1
      elif 2 == indPrediction:
        md2 = md2 + 1
      elif 3 == indPrediction:
        ms2 = ms2 + 1
  #RELAZIONE MD
  elif i < 299:
    #aggiunta previsione agli array per la matrice di confusione
    if bool2:
      fsArray.append(fd)
      fsArray.append(fs)
      fsArray.append(md)
      fsArray.append(ms)
      fsArray.append(nk)
      fd = 0
      fs = 0
      md = 0
      ms = 0
      nk = 0
      bool2 = False
    #IMMAGINI CON RELAZIONE
    if label[4] == 0:
      if 2 == indPrediction:
        count1 = count1 + 1
        md = md + 1
      elif 0 == indPrediction:
        fd = fd + 1
      elif 1 == indPrediction:
        fs = fs + 1
      elif 3 == indPrediction:
        ms = ms + 1
      elif 4 == indPrediction:
        nk = nk + 1
    #IMMAGINI NON-KIN
    else:
      if 4 == indPrediction:
        count1 = count1 + 1
        nk2 = nk2 + 1
      elif 0 == indPrediction:
        fd2 = fd2 + 1
      elif 1 == indPrediction:
        fs2 = fs2 + 1
      elif 2 == indPrediction:
        md2 = md2 + 1
      elif 3 == indPrediction:
        ms2 = ms2 + 1
  #RELAZIONE FS
  elif i < 399:
    #aggiunta previsione agli array per la matrice di confusione
    if bool3:
      mdArray.append(fd)
      mdArray.append(fs)
      mdArray.append(md)
      mdArray.append(ms)
      mdArray.append(nk)
      fd = 0
      fs = 0
      md = 0
      ms = 0
      nk = 0
      bool3 = False
    #COPPIE CON RELAZIONE
    if label[4] == 0:
      if 3 == indPrediction:
        count1 = count1 + 1
        ms = ms + 1
      elif 0 == indPrediction:
        fd = fd + 1
      elif 1 == indPrediction:
        fs = fs + 1
      elif 2 == indPrediction:
        md = md + 1
      elif 4 == indPrediction:
        nk = nk + 1
    #COPPIE NON-KIN
    else:
      if 4 == indPrediction:
        count1 = count1 + 1
        nk2 = nk2 + 1
      elif 0 == indPrediction:
        fd2 = fd2 + 1
      elif 1 == indPrediction:
        fs2 = fs2 + 1
      elif 2 == indPrediction:
        md2 = md2 + 1
      elif 3 == indPrediction:
        ms2 = ms2 + 1

    
  i = i + 1

#stampa della matrice di confusione e dell'accuratezza
print("------------")
msArray.append(fd)
msArray.append(fs)
msArray.append(md)
msArray.append(ms)
msArray.append(nk)
nkArray.append(fd2)
nkArray.append(fs2)
nkArray.append(md2)
nkArray.append(ms2)
nkArray.append(nk2)

print(fdArray)
print(fdArray)
print(mdArray)
print(msArray)
print(nkArray)
print("--------------")
print(count1)
print(len(testX))
print("ACC: (?)")
print(count1/len(testX))<jupyter_output><empty_output><jupyter_text>**MATRICE DI CONFUSIONE**Viene trasformata la matrice di confusione in un dataset, vengono rinominate righe e colonne e la si salva in un file Excel<jupyter_code>confusionMatrix = []
confusionMatrix.append(fdArray)
confusionMatrix.append(fsArray)
confusionMatrix.append(mdArray)
confusionMatrix.append(msArray)
confusionMatrix.append(nkArray)

df = pd.DataFrame([fdArray, fsArray, mdArray, msArray, nkArray], columns = ["FD", "FS", "MD", "MS", "NK"])
df = df.rename(index={0 : 'FD', 1 : "FS", 2 : "MD", 3 : "MS", 4 : "NK"})
print(df)
df.to_excel("/content/drive/MyDrive/Colab Notebooks/MatriceCovarianza.xlsx")<jupyter_output><empty_output><jupyter_text>Salvataggio predizioni in un file xml<jupyter_code>df = pd.DataFrame(np.concatenate(predictions))
df.columns = ["FD", "FS", "MD", "MS", "NK"]
df.to_excel("/content/drive/MyDrive/Colab Notebooks/vgg_faceTransferLearningExcel.xlsx")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13509997079206607, 'fraction_numerical': 0.03913863147553172, 'mean_word_length': 2.4923961424332344, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 40, 'https://': 9, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '228759', 'license': 'MIT', 'n_tokens_mistral': 14423, 'n_tokens_neox': 13414, 'n_words': 3195, 'path': 'transferLearningFINALE.ipynb', 'repo_name': 'luigi-bozzoli/siamese-neural-network', 'size': '37661'}