<jupyter_start><jupyter_text>This notebook provides the functionality to re-train, and test a CNN for predicting semi-field mosquito grouped age and species from a pre-trained model on lab data. Structure:* Import packages to be used.* Load mosquito data.* Define fucntions for plotting, visualisation, and logging.* Define a function to build the CNN.* Define a function to train the CNN.* Main section to organise data, define the CNN, and call the building and training of the CNN.<jupyter_code>import umap
import pylab as pl
import datetime
import pandas as pd
import itertools
from itertools import cycle
import pickle
import random as rn
import os
from time import time
from tqdm import tqdm

import numpy as np

import matplotlib
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D

from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import normalize, StandardScaler
from sklearn.utils import resample

import tensorflow as tf
import keras
from keras.models import Sequential, Model, load_model
from keras import layers, metrics
from keras.layers import Input
from keras.layers.merge import Concatenate
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import Conv1D, MaxPooling1D
from keras.models import model_from_json, load_model
from keras.regularizers import *
from keras.callbacks import CSVLogger
from keras import backend as K
# rand_seed = np.random.randint(low=0, high=100)
rand_seed = 16
print(rand_seed)
os.environ['PYTHONHASHSEED'] = '0'

## The below is necessary for starting Numpy generated random numbers in a well-defined initial state.
np.random.seed(42)

## The below is necessary for starting core Python generated random numbers in a well-defined state.
rn.seed(12345)

## Force TensorFlow to use single thread.
## Multiple threads are a potential source of
## non-reproducible results.
## For further details, see: https://stackoverflow.com/questions/42022950/which-seeds-have-to-be-set-where-to-realize-100-reproducibility-of-training-res
# session_conf = tf.ConfigProto(device_count = {'GPU':0}, intra_op_parallelism_threads=4) #session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
# session_conf = tf.ConfigProto(device_count = {'GPU':0}) #session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
#session_conf.gpu_options.per_process_gpu_memory_fraction = 0.5
## The below tf.set_random_seed() will make random number generation
## in the TensorFlow backend have a well-defined initial state.
## For further details, see: https://www.tensorflow.org/api_docs/python/tf/set_random_seed
tf.set_random_seed(1234)

gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)

sess = tf.Session(graph=tf.get_default_graph(), config=tf.ConfigProto(gpu_options=gpu_options))
K.set_session(sess)<jupyter_output><empty_output><jupyter_text>Load the dataThe data file is created using Loco Mosquito:https://github.com/magonji/MIMI-project/blob/master/Loco%20mosquito%204.0.ipynb The data file has headings: Species - Status - RearCnd - Age - Country- Frequencies<jupyter_code>class data_loader_class():
    def __init__(self, valid_perc, seed):
        df = pd.read_csv("/home/josh/Documents/Mosquito_Project/New_Data/Data/MIMIdata_update_19_02/mosquitoes_country_LM_5_0.dat", '\t')
        df.head(10)

        df['AgeGroup'] = 0
        df['AgeGroup'] = np.where(df['Age']>10, 2, np.where(df['Age']>4, 1, 0))

        df_vf = df[df['RearCnd']=='VF']
        df_vf = df_vf[df_vf['Status']=='UN']
        df = df[df['RearCnd']!='VF']
        df = df[df['Status']!='UN']
        df_l = df[df['RearCnd']=='TL']
        df_l_g = df_l[df_l['Country']=='S']
        
        df_l_g_a = df_l_g[df_l_g['Species']=='AA']
        age_counts = df_l_g_a.groupby('AgeGroup').size()
        df_l_g_g = df_l_g[df_l_g['Species']=='AG']
        age_counts = df_l_g_g.groupby('AgeGroup').size()
        df_l_g_c = df_l_g[df_l_g['Species']=='AC']
        age_counts = df_l_g_c.groupby('AgeGroup').size()
        df_l_t = df_l[df_l['Country']=='T']
        df_l_t_a = df_l_t[df_l_t['Species']=='AA']
        age_counts = df_l_t_a.groupby('AgeGroup').size()
        df_l_t_g = df_l_t[df_l_t['Species']=='AG']
        age_counts = df_l_t_g.groupby('AgeGroup').size()
        df_l_b = df_l[df_l['Country']=='B']
        df_l_b_g = df_l_b[df_l_b['Species']=='AG']
        age_counts = df_l_b_g.groupby('AgeGroup').size()
        df_l_b_c = df_l_b[df_l_b['Species']=='AC']
        age_counts = df_l_b_c.groupby('AgeGroup').size()
        df_f = df[df['RearCnd']=='TF']
        df_f_t = df_f[df_f['Country']=='T']
        df_f_t_a = df_f_t[df_f_t['Species']=='AA']
        # df_f_t_g = df_f_t[df_f_t['Species']=='AG'] #There isn't any
        df_f_b = df_f[df_f['Country']=='B']
        df_f_b_g = df_f_b[df_f_b['Species']=='AG']
        age_counts = df_f_b_g.groupby('AgeGroup').size()
        df_f_b_c = df_f_b[df_f_b['Species']=='AC']
        age_counts = df_f_b_c.groupby('AgeGroup').size()
        df_vf_t = df_vf[df_vf['Country']=='T']
        df_vf_t_a = df_vf_t[df_vf_t['Species']=='AA']
        age_counts = df_vf_t_a.groupby('AgeGroup').size()
        df_vf_t_g = df_vf_t[df_vf_t['Species']=='AG']
        age_counts = df_vf_t_g.groupby('AgeGroup').size()
        df_vf_b = df_vf[df_vf['Country']=='B']
        df_vf_b_g = df_vf_b[df_vf_b['Species']=='AG']
        age_counts = df_vf_b_g.groupby('AgeGroup').size()
        df_vf_b_c = df_vf_b[df_vf_b['Species']=='AC']
        age_counts = df_vf_b_c.groupby('AgeGroup').size()

        VF_size_t = len(df_vf_t)
        VF_size_b = len(df_vf_b)
        val_group_size_t = int(((((valid_perc*VF_size_t)/2)/3))) #/2 (t/b) /3 (age groups)
        val_group_size_b = int(((((valid_perc*VF_size_b)/2)/3)))


        size_test = 20
        for age in range(3):
            df_temp = df_vf_t_a[df_vf_t_a['AgeGroup']==age]
            size_df_temp = np.arange(len(df_temp))
            if len(size_df_temp) < size_test:
                print('Warning Tanzania Arabiensis VF group {} smaller than amount requested'.format(age))
            np.random.seed(seed)
            np.random.shuffle(size_df_temp)
            index_df_temp_inc = size_df_temp[:size_test]
            index_df_temp_not_inc = size_df_temp[size_test:]
            if age == 0:
                df_test = df_temp.iloc[index_df_temp_inc]
                df_train_vf_t_a = df_temp.iloc[index_df_temp_not_inc]
            else:
                df_test = pd.concat([df_test, df_temp.iloc[index_df_temp_inc]])
                df_train_vf_t_a = pd.concat([df_train_vf_t_a, df_temp.iloc[index_df_temp_not_inc]])
        for age in range(3):
            df_temp = df_vf_t_g[df_vf_t_g['AgeGroup']==age]
            size_df_temp = np.arange(len(df_temp))
            if len(size_df_temp) < size_test:
                print('Warning Tanzania Gambie VF group {} smaller than amount requested'.format(age))
            np.random.seed(seed)
            np.random.shuffle(size_df_temp)
            index_df_temp_inc = size_df_temp[:size_test]
            index_df_temp_not_inc = size_df_temp[size_test:]
            df_test = pd.concat([df_test, df_temp.iloc[index_df_temp_inc]])
            if age == 0:
                df_train_vf_t_g = df_temp.iloc[index_df_temp_not_inc]
            else:
                df_train_vf_t_g = pd.concat([df_train_vf_t_g, df_temp.iloc[index_df_temp_not_inc]])
        size_test = 10
        for age in range(3):
            df_temp = df_vf_b_g[df_vf_b_g['AgeGroup']==age]
            size_df_temp = np.arange(len(df_temp))
            if len(size_df_temp) < size_test:
                print('Warning Bobo Gambie VF group {} smaller than amount requested'.format(age))
            np.random.seed(seed)
            np.random.shuffle(size_df_temp)
            index_df_temp_inc = size_df_temp[:size_test]
            index_df_temp_not_inc = size_df_temp[size_test:]
            df_test = pd.concat([df_test, df_temp.iloc[index_df_temp_inc]])
            if age == 0:
                df_train_vf_b_g = df_temp.iloc[index_df_temp_not_inc]
            else:
                df_train_vf_b_g = pd.concat([df_train_vf_b_g, df_temp.iloc[index_df_temp_not_inc]])
        for age in range(3):
            df_temp = df_vf_b_c[df_vf_b_c['AgeGroup']==age]
            size_df_temp = np.arange(len(df_temp))
            if len(size_df_temp) < size_test:
                print('Warning Bobo Colluzzi VF group {} smaller than amount requested'.format(age))
            np.random.seed(seed)
            np.random.shuffle(size_df_temp)
            index_df_temp_inc = size_df_temp[:size_test]
            index_df_temp_not_inc = size_df_temp[size_test:]
            df_test = pd.concat([df_test, df_temp.iloc[index_df_temp_inc]])
            if age == 0:
                df_train_vf_b_c = df_temp.iloc[index_df_temp_not_inc]
            else:
                df_train_vf_b_c = pd.concat([df_train_vf_b_c, df_temp.iloc[index_df_temp_not_inc]])


        size_inc = val_group_size_t
        for age in range(3):
            df_temp = df_train_vf_t_a[df_train_vf_t_a['AgeGroup']==age]
            size_df_temp = np.arange(len(df_temp))
            if len(size_df_temp) < size_inc:
                print('Warning Tanzania Arabiensis VF group {} smaller than amount requested'.format(age))
            np.random.seed(seed)
            np.random.shuffle(size_df_temp)
            index_df_temp_inc = size_df_temp[:size_inc]
            index_df_temp_not_inc = size_df_temp[size_inc:]
#             df_train = df_temp.iloc[index_df_temp_inc]#pd.concat([df_train, df_temp.iloc[index_df_temp_inc]])
            if age == 0:
                df_train = df_temp.iloc[index_df_temp_inc]
#                 df_test = df_temp.iloc[index_df_temp_not_inc]
            else:
                df_train = pd.concat([df_train, df_temp.iloc[index_df_temp_inc]])
#                 df_test = pd.concat([df_test, df_temp.iloc[index_df_temp_not_inc]])
        for age in range(3):
            df_temp = df_train_vf_t_g[df_train_vf_t_g['AgeGroup']==age]
            size_df_temp = np.arange(len(df_temp))
            if len(size_df_temp) < size_inc:
                print('Warning Tanzania Gambie VF group {} smaller than amount requested'.format(age))
            np.random.seed(seed)
            np.random.shuffle(size_df_temp)
            index_df_temp_inc = size_df_temp[:size_inc]
            index_df_temp_not_inc = size_df_temp[size_inc:]
            df_train = pd.concat([df_train, df_temp.iloc[index_df_temp_inc]])
#             df_test = pd.concat([df_test, df_temp.iloc[index_df_temp_not_inc]])
        size_inc = val_group_size_b
        for age in range(3):
            df_temp = df_train_vf_b_g[df_train_vf_b_g['AgeGroup']==age]
            size_df_temp = np.arange(len(df_temp))
            if len(size_df_temp) < size_inc:
                print('Warning Bobo Gambie VF group {} smaller than amount requested'.format(age))
            np.random.seed(seed)
            np.random.shuffle(size_df_temp)
            index_df_temp_inc = size_df_temp[:size_inc]
            index_df_temp_not_inc = size_df_temp[size_inc:]
            df_train = pd.concat([df_train, df_temp.iloc[index_df_temp_inc]])
#             df_test = pd.concat([df_test, df_temp.iloc[index_df_temp_not_inc]])
        for age in range(3):
            df_temp = df_train_vf_b_c[df_train_vf_b_c['AgeGroup']==age]
            size_df_temp = np.arange(len(df_temp))
            if len(size_df_temp) < size_inc:
                print('Warning Bobo Colluzzi VF group {} smaller than amount requested'.format(age))
            np.random.seed(seed)
            np.random.shuffle(size_df_temp)
            index_df_temp_inc = size_df_temp[:size_inc]
            index_df_temp_not_inc = size_df_temp[size_inc:]
            df_train = pd.concat([df_train, df_temp.iloc[index_df_temp_inc]])
#             df_test = pd.concat([df_test, df_temp.iloc[index_df_temp_not_inc]])

        print('Percentage of field mosquitoes inc {} - Num mosquitoes {} / {}'.format(valid_perc*100, len(df_train[df_train['RearCnd']=='VF']), len(df_vf)))
        print('Total number of mosquitoes in the Train set {}'.format(len(df_train)))
        print('Total number of mosquitoes in the Test set {}'.format(len(df_test)))
        
        X = df_train.iloc[:,6:-1]
        y_age = df_train["Age"]
        y_age_groups = df_train["AgeGroup"]
        y_species = df_train["Species"]
        y_status = df_train["Status"]
#         print('shape of X : {}'.format(X.shape))
#         print('shape of y age : {}'.format(y_age.shape))
#         print('shape of y age groups : {}'.format(y_age_groups.shape))
#         print('shape of y species : {}'.format(y_species.shape))
#         print('shape of y status : {}'.format(y_status.shape))
        self.X = np.asarray(X)
        y_age = np.asarray(y_age)
        self.y_age_groups = np.asarray(y_age_groups)
        self.y_species = np.asarray(y_species)
        y_status = np.asarray(y_status)

        X_vf = df_test.iloc[:,6:-1]
        y_age_vf = df_test["Age"]
        y_age_groups_vf = df_test["AgeGroup"]
        y_species_vf = df_test["Species"]
        y_status_vf = df_test["Status"]
#         print('shape of X_vf : {}'.format(X_vf.shape))
#         print('shape of y_age_vf age : {}'.format(y_age_vf.shape))
#         print('shape of y_age_groups_vf : {}'.format(y_age_groups_vf.shape))
#         print('shape of y y_species_vf : {}'.format(y_species_vf.shape))
#         print('shape of y y_status_vf : {}'.format(y_status_vf.shape))
        self.X_vf = np.asarray(X_vf)
        y_age_vf = np.asarray(y_age_vf)
        self.y_age_groups_vf = np.asarray(y_age_groups_vf)
        self.y_species_vf = np.asarray(y_species_vf)
        y_status_vf = np.asarray(y_status_vf)


def data_loader(valid_perc, seed):
    return data_loader_class(valid_perc, seed)
for valid_inc_perc in [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]:
    data_loader(valid_inc_perc, 0)
    print('------------------------------------------------------------')<jupyter_output><empty_output><jupyter_text>Function used to create a new folder for the CNN outputs.Useful to stop forgetting to name a new folder when trying out a new model varient and overwriting a days training.<jupyter_code>def build_folder(fold, to_build = False):
    if not os.path.isdir(fold):
        if to_build == True:
            os.mkdir(fold)
        else:
            print('Directory does not exists, not creating directory!')
    else:
        if to_build == True:
            raise NameError('Directory already exists, cannot be created!')<jupyter_output><empty_output><jupyter_text>Function for plotting confusion matrciesThis normalizes the confusion matrix and ensures neat plotting for all outputs.<jupyter_code>def plot_confusion_matrix(cm, classes, output, save_path, model_name, fold,
                          normalize=True,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues,
                          printout=False):

    font = {'weight' : 'normal',
            'size'   : 18}

    matplotlib.rc('font', **font)

    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        if printout:
            print("Normalized confusion matrix")
    else:
        if printout:
            print('Confusion matrix, without normalization')

    if printout:
        print(cm)
    
    plt.figure(figsize=(8,8))
    plt.imshow(cm, interpolation='nearest', cmap=cmap, vmin=0, vmax=1) # np.max(np.sum(cm, axis=1)))
#     plt.title([title+' - '+model_name])
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)
    plt.ylim(2.5,-0.5)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout(pad=2)
#     plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.savefig((save_path+"Confusion_Matrix_"+model_name+"_"+fold+"_"+output[1:]+".pdf"))
    plt.close()<jupyter_output><empty_output><jupyter_text>Function used for visualizing outputsThis splits the output data into the four categories before plotting the confusion matricies.<jupyter_code>## for visualizing losses and metrics once the neural network fold is trained
def visualize(histories, save_path, model_name, fold, classes, outputs, predicted, true, title='Confusion Matrix'):
    # Sort out predictions and true labels
    for label_predictions_arr, label_true_arr, classes, outputs in zip(predicted, true, classes, outputs):
        classes_pred = np.argmax(label_predictions_arr, axis=-1)
        classes_true = np.argmax(label_true_arr, axis=-1)
        cnf_matrix = confusion_matrix(classes_true, classes_pred)
        plot_confusion_matrix(cnf_matrix, classes, outputs, save_path, model_name, fold, title=title)<jupyter_output><empty_output><jupyter_text>Data logging<jupyter_code>## for logging data associated with the model
def log_data(log, name, fold, save_path):
    f = open((save_path+name+'_'+str(fold)+'_log.txt'), 'w')
    np.savetxt(f, log)
    f.close()
def save_obj(obj, name, savedir_main):
    with open(savedir_main + name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
def load_obj(name, savedir_main):
    with open(savedir_main + name + '.pkl', 'rb') as f:
        return pickle.load(f)<jupyter_output><empty_output><jupyter_text>Fucntion for graphing the training dataThis fucntion creates tidy graphs of loss and accuracy as the models are training.<jupyter_code>def graph_history(history, model_name, model_ver_num, fold, save_path):

    font = {'weight' : 'normal',
            'size'   : 18}

    matplotlib.rc('font', **font)
    
    #not_validation = list(filter(lambda x: x[0:3] != "val", history.history.keys()))
#     print('history.history.keys : {}'.format(history.history.keys()))
    filtered = filter(lambda x: x[0:3] != "val", history.history.keys())
    not_validation = list(filtered)
    for i in not_validation:
        plt.figure(figsize=(15,7))
#         plt.title(i+"/ "+"val_"+i)
        plt.plot(history.history[i], label=i)
        plt.plot(history.history["val_"+i], label="val_"+i)
        plt.legend()
        plt.xlabel("epoch")
        plt.ylabel(i)
        plt.savefig(save_path +model_name+"_"+str(model_ver_num)+"_"+str(fold)+"_"+i+".png")
        plt.close()<jupyter_output><empty_output><jupyter_text>funciton to create the CNNThis function takes as an input a list of dictionaries. Each element in the list is a new hidden layer in the model. For each layer the dictionary defines the layer to be used. Available options are:Convolutional Layer:* type = 'c'* filter = optional number of filters* kernel = optional size of the filters* stride = optional size of stride to take between filters* pooling = optional width of the max pooling* {'type':'c', 'filter':16, 'kernel':5, 'stride':1, 'pooling':2}dense layer:* type = 'd'* width = option width of the layer* {'type':'d', 'width':500}<jupyter_code>def create_models(model_shape, input_layer):

    regConst = 0.02
    sgd = keras.optimizers.SGD(lr=0.003, decay=1e-5, momentum=0.9, nesterov=True, clipnorm=1.)
    cce = 'categorical_crossentropy'

    input_vec = Input(name='input', shape=(input_layer_dim,1))

    for i, layerwidth in zip(range(len(model_shape)),model_shape):
        if i == 0:
            if model_shape[i]['type'] == 'c':
                xd = Conv1D(name=('Conv'+str(i+1)), filters=model_shape[i]['filter'], 
                 kernel_size = model_shape[i]['kernel'], strides = model_shape[i]['stride'],
                 activation = 'relu',
                 kernel_regularizer=l2(regConst), 
                 kernel_initializer='he_normal')(input_vec)
                xd = BatchNormalization(name=('batchnorm_'+str(i+1)))(xd)
                xd = MaxPooling1D(pool_size=(model_shape[i]['pooling']))(xd)
                
            elif model_shape[i]['type'] == 'd':
                xd = Dense(name=('d'+str(i+1)), units=model_shape[i]['width'], activation='relu', 
                 kernel_regularizer=l2(regConst), 
                 kernel_initializer='he_normal')(input_vec)
                xd = BatchNormalization(name=('batchnorm_'+str(i+1)))(xd) 
                xd = Dropout(name=('dout'+str(i+1)), rate=0.5)(xd) 
                
        else:
            if model_shape[i]['type'] == 'c':
                xd = Conv1D(name=('Conv'+str(i+1)), filters=model_shape[i]['filter'], 
                 kernel_size = model_shape[i]['kernel'], strides = model_shape[i]['stride'],
                 activation = 'relu',
                 kernel_regularizer=l2(regConst), 
                 kernel_initializer='he_normal')(xd)
                xd = BatchNormalization(name=('batchnorm_'+str(i+1)))(xd)
                xd = MaxPooling1D(pool_size=(model_shape[i]['pooling']))(xd)
                
            elif model_shape[i]['type'] == 'd':
                if model_shape[i-1]['type'] == 'c':
                    xd = Flatten()(xd)
                    
                xd = Dropout(name=('dout'+str(i+1)), rate=0.5)(xd)
                xd = Dense(name=('d'+str(i+1)), units=model_shape[i]['width'], activation='relu', 
                 kernel_regularizer=l2(regConst), 
                 kernel_initializer='he_normal')(xd)
                xd = BatchNormalization(name=('batchnorm_'+str(i+1)))(xd) 
        
    
    xAgeGroup     = Dense(name = 'age_group', units = 3, 
                     activation = 'softmax', 
                     kernel_regularizer = l2(regConst), 
                     kernel_initializer = 'he_normal')(xd)
    xSpecies = Dense(name ='species', units = 3, 
                     activation = 'softmax', 
                     kernel_regularizer = l2(regConst), 
                     kernel_initializer = 'he_normal')(xd)

    outputs = []
    for i in ['xAgeGroup', 'xSpecies']:
        outputs.append(locals()[i])
    model = Model(inputs = input_vec, outputs = outputs)
    
    model.compile(loss=cce, metrics=['acc'], 
                  optimizer=sgd)
#     model.summary()
    return model<jupyter_output><empty_output><jupyter_text>Function to train the modelThis function will split the data into training and validation and call the create models function. This fucntion returns the model and training history.<jupyter_code>def train_models(model_to_test, save_path, SelectFreqs=False):

    model_shape = model_to_test["model_shape"][0]
    model_name = model_to_test["model_name"][0]
    model_ver_num = model_to_test["model_ver_num"][0]
    fold = model_to_test["fold"][0]
    label = model_to_test["labels"][0]
    features = model_to_test["features"][0]
    classes = model_to_test["classes"][0]
    outputs = model_to_test["outputs"][0]
    compile_loss = model_to_test["compile_loss"][0]
    compile_metrics = model_to_test["compile_metrics"][0]

    ## Split into training / testing
    test_splits = train_test_split(features, *(label), test_size=0.1, shuffle=True, random_state=rand_seed)
    ## Pack up data
    X_train = test_splits.pop(0)
    X_val = test_splits.pop(0)
    y_train = test_splits[::2]
    y_val = test_splits[1::2]
    
    model = load_model("/home/josh/Documents/Mosquito_Project/MIMI-Analysis/Models/Neural_Networks/CNN/Paper/Results_Paper/Trian_Lab_Field_V2/Baseline_CNN_Model.h5")
    model.summary()
    ## Freeze CNN layers
    for layer in model.layers[:-5]:
        layer.trainable = False
    for layer in model.layers:
        print(f'{layer.name} : {layer.trainable}')
        
    history = model.fit(x = X_train, 
                            y = y_train,
                            batch_size = 128*16, 
                            verbose = 0, 
                            epochs = 8000,
                            validation_data = (X_val, y_val),
                            callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss', 
                                        patience=400, verbose=0, mode='auto'), 
                                        CSVLogger(save_path+model_name+"_"+str(model_ver_num)+'.csv', append=True, separator=';')])
    scores = model.evaluate(X_val, y_val)
        
    model.save((save_path+model_name+"_"+'Model.h5'))
    graph_history(history, model_name, 0, 0, save_path)
    
    return model, history
def load_only_models():

    model = load_model("/home/josh/Documents/Mosquito_Project/MIMI-Analysis/Models/Neural_Networks/CNN/Paper/Results_Paper/Trian_Lab_Field_V2/Baseline_CNN_Model.h5")
    model.summary()
    for layer in model.layers[:-5]:
        layer.trainable = False
    for layer in model.layers:
        print(f'{layer.name} : {layer.trainable}')
    
    return model<jupyter_output><empty_output><jupyter_text>Main sectionFunctionality:* Oganises the data into a format of lists of data, classes, labels.* Define the CNN to be built.* Define the KFold validation to be used.* Build a folder to output data into.* Standardize and oragnise data into training/testing.* Call the model training.* Organize outputs and call visualization for plotting and graphing.<jupyter_code>## Name a folder for the outputs to go into
outdir = "Results_Paper/"
build_folder(outdir, False)

savedir_main = (outdir+"Trian_Val_Only_Fix_Test_Retrain_Repeats/")
build_folder(savedir_main, False)

val_results = {'loss':[], 
               'age_group_loss':[], 
               'species_loss':[], 
               'age_group_acc':[], 
               'species_acc':[]}
histories = []

start_time = time()

for idx, valid_inc_perc in enumerate([0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]):
    
    savedir_1 = (savedir_main+"valid_data_inc_"+str(valid_inc_perc)+"/")
    build_folder(savedir_1, True)
    
    for repeat in range(20):
        
        savedir = (savedir_1+"repeat_"+str(repeat)+"/")
        build_folder(savedir, True)

        data_extract = data_loader(valid_inc_perc, seed=repeat)


        ## Transform Data
        y_age_groups_list = [[age] for age in data_extract.y_age_groups]
        y_species_list = [[species] for species in data_extract.y_species]
        age_groups = MultiLabelBinarizer().fit_transform(np.array(y_age_groups_list))
        age_group_classes = ["1-4", "5-10", "11-17"]
        species = MultiLabelBinarizer().fit_transform(np.array(y_species_list))
        species_classes = list(np.unique(y_species_list))
        y_age_groups_list_vf = [[age] for age in data_extract.y_age_groups_vf]
        y_species_list_vf = [[species] for species in data_extract.y_species_vf]
        age_groups_vf = MultiLabelBinarizer().fit_transform(np.array(y_age_groups_list_vf))
        species_vf = MultiLabelBinarizer().fit_transform(np.array(y_species_list_vf))

        ## Labels
        labels_default, classes_default, outputs_default = [age_groups, species], [age_group_classes, species_classes], ['xAgeGroup', 'xSpecies']
        labels_default_vf, classes_default_vf, outputs_default_vf = [age_groups_vf, species_vf], [age_group_classes, species_classes], ['xAgeGroup', 'xSpecies']

        ## Declare and train the model
        model_size = [{'type':'c', 'filter':16, 'kernel':8, 'stride':1, 'pooling':1},
                     {'type':'c', 'filter':16, 'kernel':8, 'stride':2, 'pooling':1},
                     {'type':'c', 'filter':16, 'kernel':3, 'stride':1, 'pooling':1},
                     {'type':'c', 'filter':16, 'kernel':6, 'stride':2, 'pooling':1},
                     {'type':'c', 'filter':16, 'kernel':5, 'stride':1, 'pooling':2},
                     {'type':'d', 'width':500}]

        ## Name the model
        model_name = ('Valid_Inc_'+str(valid_inc_perc))

        ## Scale train, test
        scl = StandardScaler()
        features_scl = scl.fit(X=np.vstack((data_extract.X, data_extract.X_vf)))
        if valid_inc_perc != 0:
            X_train = features_scl.transform(X=data_extract.X)
        X_test = features_scl.transform(X=data_extract.X_vf)

        ## Split data into test and train
        y_train, y_test = list(map(lambda y:y, labels_default)), list(map(lambda y:y, labels_default_vf))

        if valid_inc_perc != 0:
            X_train = np.expand_dims(X_train, axis=2)
        X_test = np.expand_dims(X_test, axis=2)

        if valid_inc_perc != 0:
            model_to_test = {
                "model_shape" : [model_size], # defines the hidden layers of the model
                "model_name"  : [model_name],
                "model_ver_num"  : [0],
                "fold"  : [0],
                "labels"   : [y_train],
                "features" : [X_train],
                "classes"  : [classes_default],
                "outputs"   : [outputs_default],
                "compile_loss": [{'age': 'categorical_crossentropy'}],
                "compile_metrics" :[{'age': 'accuracy'}]
            }                

        ## Call function to train all the models from the dictionary
        if valid_inc_perc == 0:
            model = load_only_models()
        else:
            model, history = train_models(model_to_test, savedir)
            histories.append(history)
        
        
        predicted_labels = list([] for i in range(len(y_train)))
        true_labels = list([] for i in range(len(y_train)))

        y_predicted = model.predict(X_test)
        temp_eval = model.evaluate(X_test, y_test)
        for metric, res in zip(model.metrics_names, temp_eval):
            val_results[metric].append(res)
        print(val_results)

        predicted_labels = [x+[y] for x,y in zip(predicted_labels,y_predicted)]
        true_labels = [x+[y] for x,y in zip(true_labels,y_test)]

        predicted_labels = [predicted_labels[i][0].tolist() for i in range(len(predicted_labels))]
        true_labels = [true_labels[i][0].tolist() for i in range(len(true_labels))]

        ## Visualize the results
        visualize(histories, savedir, model_name, "0", classes_default, outputs_default, predicted_labels, true_labels)

        # Clear the Keras session, otherwise it will keep adding new
        # models to the same TensorFlow graph each time we create
        # a model with a different set of hyper-parameters.
        K.clear_session()

        # Delete the Keras model with these hyper-parameters from memory.
        del model

end_time = time()
print('Run time : {} s'.format(end_time-start_time))
print('Run time : {} m'.format((end_time-start_time)/60))
print('Run time : {} h'.format((end_time-start_time)/3600))

save_obj(val_results, f'Validation_Results_Dict_{valid_inc_perc}', savedir_main)
plt.rcParams.update({'font.size': 15})

outdir = "Results_Paper/"
build_folder(outdir, False)

savedir_main = (outdir+"Trian_Val_Only_Fix_Test_Retrain_Repeats/")
build_folder(savedir_main, False)

val_results = {'loss':[], 
               'age_group_loss':[], 
               'species_loss':[], 
               'age_group_acc':[], 
               'species_acc':[]}
for i in [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45]:
    val_results_load = load_obj(f'Validation_Results_Dict_{i}', savedir_main)
    val_results['loss'].append(val_results_load['loss'])
    val_results['age_group_loss'].append(val_results_load['age_group_loss'])
    val_results['species_loss'].append(val_results_load['species_loss'])
    val_results['age_group_acc'].append(val_results_load['age_group_acc'])
    val_results['species_acc'].append(val_results_load['species_acc'])
    
x = [0, 162, 324, 486, 654, 815, 973, 1131, 1294, 1452]

fig = plt.figure(figsize=(5.6,2.8))
ax = fig.add_subplot(111)
age_group_acc = np.array(val_results["age_group_acc"])
species_acc = np.array(val_results["species_acc"])
means_age = np.mean(age_group_acc, axis=1)
means_species = np.mean(species_acc, axis=1)
stds_age = np.std(age_group_acc, axis=1)
stds_species = np.std(species_acc, axis=1)
ax.plot(x, means_age, lw=1.5, c='b', label="Age Group Acc")
ax.fill_between(x, means_age-stds_age, means_age+stds_age, color='b', alpha=0.2)
ax.plot(x, means_species, lw=1.5, c='g', label="Species Acc")
ax.fill_between(x, means_species-stds_species, means_species+stds_species, color='g', alpha=0.2)
ax.legend(loc='lower right')
ax.set_xlabel("Number of Semifeld Datapoints")
ax.set_ylabel("Test Accuracy")
ax.set_xlim([0,1500])
ax.set_ylim([0,1])
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
ax.grid(True, axis='y', which='both')
plt.tight_layout(pad=1)

plt.savefig(savedir_main + "Validation_Study.png")
plt.savefig(savedir_main + "train-from-lab-field-model-EV-only.pdf")
plt.close()
plt.rcParams.update({'font.size': 20})

outdir = "Results_Paper/"
build_folder(outdir, False)

savedir_main = (outdir+"Trian_Val_Only_Fix_Test_Retrain_Repeats/")
build_folder(savedir_main, False)

val_results = {'loss':[], 
               'age_group_loss':[], 
               'species_loss':[], 
               'age_group_acc':[], 
               'species_acc':[]}
for i in [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45]:
    val_results_load = load_obj(f'Validation_Results_Dict_{i}', savedir_main)
    val_results['loss'].append(val_results_load['loss'])
    val_results['age_group_loss'].append(val_results_load['age_group_loss'])
    val_results['species_loss'].append(val_results_load['species_loss'])
    val_results['age_group_acc'].append(val_results_load['age_group_acc'])
    val_results['species_acc'].append(val_results_load['species_acc'])
    
x = [0, 162, 324, 486, 654, 815, 973, 1131, 1294, 1452]

fig = plt.figure(figsize=(15,7))
ax = fig.add_subplot(111)
age_group_acc = val_results["age_group_acc"]
species_acc = val_results["species_acc"]

ax.boxplot(age_group_acc, positions=[0,1,2,3,4,5,6,7,8,9], labels=x)

# ax.legend(loc='lower right')
ax.set_xlabel("Number of Semifeld Datapoints in the Total Training Dataset")
ax.set_ylabel("Test Accuracy")
# ax.set_xticks([0, 2.2, 4.3, 6.3, 8.3, 10.2, 11.9, 13.6, 15.2, 16.8])
ax.set_xlim(left=0)
ax.set_ylim([0,1])
plt.savefig(savedir_main + "train-from-lab-field-model-EV-only-BP.pdf")
plt.close()<jupyter_output><empty_output><jupyter_text>Plotting with Validation + Lab/Field experiment<jupyter_code>plt.rcParams.update({'font.size': 20})

outdir = "Results_Paper/"
build_folder(outdir, False)

savedir_main = (outdir+"Trian_Val_Only2/")
build_folder(savedir_main, False)

val_results = load_obj('Validation_Results_Dict', savedir_main)
print(val_results["age_group_acc"])
x = [162, 324, 486, 654, 815, 973, 1131, 1294, 1452]

loaddir_main = (outdir+"Trian_Val_Inc/")
build_folder(loaddir_main, False)
val_results_1 = load_obj('Validation_Results_Dict', loaddir_main)
print(val_results_1["age_group_acc"])
x_1 = [0, 162, 324, 486, 654, 815, 973, 1131, 1294, 1452]

fig = plt.figure(figsize=(15,7))
ax = fig.add_subplot(111)
ax.plot(x_1, val_results_1["age_group_acc"][:-1], lw=2, c='b', marker='o', markersize=10, label="Age Group")
ax.plot(x_1, val_results_1["species_acc"][:-1], lw=2, c='g', marker='o', markersize=10, label="Species")
ax.plot(x, val_results["age_group_acc"][:-1], lw=2, c='c', marker='o', markersize=10, label="Age Group - SemiField Only")
ax.plot(x, val_results["species_acc"][:-1], lw=2, c='m', marker='o', markersize=10, label="Species - SemiField Only")
ax.legend(loc='lower right')
ax.set_xlabel("Number of Semifeld Datapoints in the Total Training Dataset (+ 7200 Non-Semifield Datapoints)")
ax.set_ylabel("Test Accuracy")
ax.set_xlim(left=0)
ax.set_ylim([0,1])

plt.savefig(savedir_main + "Validation_Study_Both.png")
plt.close()
plt.rcParams.update({'font.size': 20})

outdir = "Results_Paper/"
build_folder(outdir, False)

savedir_main = (outdir+"Trian_Val_Only2/")
build_folder(savedir_main, False)

val_results = load_obj('Validation_Results_Dict', savedir_main)
print(val_results["age_group_acc"])
x = [162, 324, 486, 654, 815, 973, 1131, 1294, 1452]

loaddir_main = (outdir+"Trian_Val_Inc/")
build_folder(loaddir_main, False)
val_results_1 = load_obj('Validation_Results_Dict', loaddir_main)
print(val_results_1["age_group_acc"])
x_1 = [162, 324, 486, 654, 815, 973, 1131, 1294, 1452]

fig = plt.figure(figsize=(15,7))
ax = fig.add_subplot(111)
ax.plot(x_1, (val_results_1["age_group_acc"][1:-1]-val_results_1["age_group_acc"][1])*100, lw=2, c='b', marker='o', markersize=10, label="Age Group (EV + 7200 LV+GV Datapoints)")
ax.plot(x_1, (val_results_1["species_acc"][1:-1]-val_results_1["species_acc"][1])*100, lw=2, c='g', marker='o', markersize=10, label="Species (EV + 7200 LV+GV Datapoints)")
ax.plot(x, (val_results["age_group_acc"][:-1]-val_results["age_group_acc"][0])*100, lw=2, c='c', marker='o', markersize=10, label="Age Group - EV Only")
ax.plot(x, (val_results["species_acc"][:-1]-val_results["species_acc"][0])*100, lw=2, c='m', marker='o', markersize=10, label="Species - EV Only")
ax.set_xlabel("Number of EV Datapoints in the Total Training Dataset")
ax.set_ylabel("Test Accuracy Gain (%)")
ax.set_xlim(left=0)
ax.set_ylim([0,38])

plt.rcParams.update({'font.size': 18})
ax.legend(loc='upper left')
plt.savefig(savedir_main + "Validation_Study_Both_Impr.png")
plt.close()<jupyter_output>[0.548635633990623, 0.6244497121974972, 0.6950913651021139, 0.7479984751422124, 0.7847278632834904, 0.7764756944444444, 0.8178005591798695, 0.8593040848102945, 0.8668493150684932, 0.8679135009463526]
[0.34330180042722, 0.5338683788121991, 0.6352861496782932, 0.6864922966678609, 0.7483797177780325, 0.7806661254404813, 0.8038194444444444, 0.8355079217148182, 0.8567826522761262, 0.8717808219178083, 0.8965517242424395]<jupyter_text>UMAP visualisation of CNN embedding<jupyter_code>## Name a folder for the outputs to go into
outdir = "Results_Paper/"
build_folder(outdir, False)

savedir_main = (outdir+"Trian_Val_Only_Fix_Test_Retrain_Repeats/")
build_folder(savedir_main, False)

model = load_model("/home/josh/Documents/Mosquito_Project/MIMI-Analysis/Models/Neural_Networks/CNN/Paper/Results_Paper/Trian_Lab_Field_V2/Baseline_CNN_Model.h5")
model.summary()

inputs = model.input
predictions = model.layers[-5].output
model = Model(inputs=inputs, outputs=predictions)
sgd = keras.optimizers.SGD(lr=0.003, decay=1e-5, momentum=0.9, nesterov=True, clipnorm=1.)
cce = 'categorical_crossentropy'
model.compile(loss=cce, metrics=['acc'], 
                  optimizer=sgd)
# df = pd.read_csv("/home/josh/Documents/Mosquito_Project/New_Data/Data/MIMIdata/mosquitos_country.dat", '\t')
df = pd.read_csv("/home/josh/Documents/Mosquito_Project/New_Data/Data/MIMIdata_update_19_02/mosquitoes_country_LM_5_0.dat", '\t')
df.head(10)

df = df[df['Country']!='VB']

df['AgeGroup'] = 0
df['AgeGroup'] = np.where(df['Age']>10, 2, np.where(df['Age']>4, 1, 0))

Species_counts = df.groupby('Species').size()
print('{}'.format(Species_counts))
Status_counts = df.groupby('Status').size()
print('{}'.format(Status_counts))
RearCnd_counts = df.groupby('RearCnd').size()
print('{}'.format(RearCnd_counts))
age_counts = df.groupby('Age').size()
print('{}'.format(age_counts))
country_counts = df.groupby('Country').size()
print('{}'.format(country_counts))

X = df.iloc[:,6:-1]
y_age = df["Age"]
y_species = df["Species"]
y_status = df["Status"]
y_country = df["Country"]
y_age_groups = df["AgeGroup"]
y_rearcnd = df["RearCnd"]


print('shape of X : {}'.format(X.shape))
print('shape of y age : {}'.format(y_age.shape))
print('shape of y species : {}'.format(y_species.shape))
print('shape of y status : {}'.format(y_status.shape))
print('shape of y country : {}'.format(y_country.shape))
print('shape of y age groups : {}'.format(y_age_groups.shape))
print('shape of y rearcnd : {}'.format(y_rearcnd.shape))

X = np.asarray(X)
y_age = np.asarray(y_age)
y_species = np.asarray(y_species)
y_status = np.asarray(y_status)
y_country = np.asarray(y_country)
y_age_groups = np.asarray(y_age_groups)
y_rearcnd = np.asarray(y_rearcnd)
def plot_embedding_country(X, y, Species=None, saveDir=None, title=None, colorMap='Greys'):
    ## Sort the data
    x_min, x_max = np.min(X, 0), np.max(X, 0)
    X = (X - x_min) / (x_max - x_min)

    if Species is not None:
        uniqueSpecies = np.unique(Species)
    
    ## Start plotting
    colors = ['r', 'b', 'g', 'k']
    y_color = []
    y_label = []
    unique_y = np.unique(y)
    for i in y:
        lab_val = 0
        for lab in unique_y:
            if i == lab:
                y_color.append(colors[lab_val])
                if lab == 'S':
                    y_label.append('Glasgow')
                elif lab == 'T':
                    y_label.append('Ifakara')
                elif lab == 'B':
                    y_label.append('Bobo')
            lab_val += 1
    y_color = np.asarray(y_color)
    y_label = np.asarray(y_label)
    
    fig = plt.figure(figsize=(5.6,2.8))

    for i in range(len(uniqueSpecies)):
        ax = plt.subplot(1,len(uniqueSpecies),i+1)
        ## Get indicies of the species
        S = []
        L = []
        for j in range(len(unique_y)):
            indicies = np.where(np.logical_and(Species==uniqueSpecies[i], y==unique_y[j]))[0]
            if not list(indicies):
                pass
            else:
                S1 = plt.scatter(X[indicies, 0], X[indicies, 1], c=y_color[indicies], label=y_label[indicies], s=0.1)       
                S.append(S1)
            if not list(y_label[indicies]):
                pass
            else:
                L.append(list(y_label[indicies])[0])

        plt.xticks([]), plt.yticks([])
        if uniqueSpecies[i] =='AA':
            plt.title('An. Arabiensis', fontsize=15)
        elif uniqueSpecies[i] =='AC':
            plt.title('An. Coluzzii', fontsize=15)
        elif uniqueSpecies[i] =='AG':
            plt.title('An. Gambiae', fontsize=15)

    plt.savefig(saveDir+title+".pdf")
    
    fig = plt.figure(figsize=(5.6,2.8))

    for i in range(len(uniqueSpecies)):
        ax = plt.subplot(1,len(uniqueSpecies),i+1)
        ## Get indicies of the species
        S = []
        L = []
        for j in range(len(unique_y)):
            indicies = np.where(np.logical_and(Species==uniqueSpecies[i], y==unique_y[j]))[0]
            if not list(indicies):
                pass
            else:
                S1 = plt.scatter(X[indicies, 0], X[indicies, 1], c=y_color[indicies], label=y_label[indicies], s=5)       
                S.append(S1)
            if not list(y_label[indicies]):
                pass
            else:
                L.append(list(y_label[indicies])[0])

        plt.xticks([]), plt.yticks([])

    fig = plt.figure(figsize=(5.6,2.8))
    fig.legend((S), (L), loc='lower center', fontsize=15, markerscale=4.)      

    plt.savefig(saveDir+title+"_legend.pdf", bbox_inches="tight")
def plot_embedding_rearCnd(X, y, Species=None, saveDir=None, title=None, colorMap='Greys'):
    ## Sort the data
    x_min, x_max = np.min(X, 0), np.max(X, 0)
    X = (X - x_min) / (x_max - x_min)

    if Species is not None:
        uniqueSpecies = np.unique(Species)
    
    ## Start plotting
    colors = ['r', 'b', 'g', 'o', 'p']
    y_color = []
    y_label = []
    unique_y = np.unique(y)
    for i in y:
        lab_val = 0
        for lab in unique_y:
            if i == lab:
                y_color.append(colors[lab_val])
                if lab == 'TF':
                    y_label.append('Time Course Field')
                elif lab == 'TL':
                    y_label.append('Time Course Lab')
                elif lab == 'VF':
                    y_label.append('Validation Field')
            lab_val += 1
    y_color = np.asarray(y_color)
    y_label = np.asarray(y_label)
    print(f'y_color shape : {y_color.shape}')
    print(f'y_label shape : {y_label.shape}')
    
    fig = plt.figure(figsize=(5.6,2.8))
    for i in range(len(uniqueSpecies)):
        ax = plt.subplot(1,len(uniqueSpecies),i+1)
        ## Get indicies of the species
        S = []
        L = []
        for j in range(len(unique_y)):
            indicies = np.where(np.logical_and(Species==uniqueSpecies[i], y==unique_y[j]))[0]
            if not list(indicies):
                pass
            else:
                S1 = plt.scatter(X[indicies, 0], X[indicies, 1], c=y_color[indicies], label=y_label[indicies], s=0.1)       
                S.append(S1)
            if not list(y_label[indicies]):
                pass
            else:
                L.append(list(y_label[indicies])[0])

        plt.xticks([]), plt.yticks([])
        if uniqueSpecies[i] =='AA':
            plt.title('An. Arabiensis', fontsize=15)
        elif uniqueSpecies[i] =='AC':
            plt.title('<NAME>', fontsize=15)
        elif uniqueSpecies[i] =='AG':
            plt.title('An. Gambiae', fontsize=15)

    plt.savefig(saveDir+title+".pdf")
    
    fig = plt.figure(figsize=(5.6,2.8))
    for i in range(len(uniqueSpecies)):
        ax = plt.subplot(1,len(uniqueSpecies),i+1)
        ## Get indicies of the species
        S = []
        L = []
        for j in range(len(unique_y)):
            indicies = np.where(np.logical_and(Species==uniqueSpecies[i], y==unique_y[j]))[0]
            if not list(indicies):
                pass
            else:
                S1 = plt.scatter(X[indicies, 0], X[indicies, 1], c=y_color[indicies], label=y_label[indicies], s=5)       
                S.append(S1)
            if not list(y_label[indicies]):
                pass
            else:
                L.append(list(y_label[indicies])[0])

        plt.xticks([]), plt.yticks([])
        if uniqueSpecies[i] =='AA':
            plt.title('An. Arabiensis', fontsize=15)
        elif uniqueSpecies[i] =='AC':
            plt.title('An. Coluzzii', fontsize=15)
        elif uniqueSpecies[i] =='AG':
            plt.title('An. Gambiae', fontsize=15)

    fig = plt.figure(figsize=(5.6,2.8))
    fig.legend((S), (L), loc='lower center', fontsize=18, markerscale=4.)     

    plt.savefig(saveDir+title+"_legend.pdf", bbox_inches="tight")
def plot_embedding_age(X, y, Species=None, saveDir=None, title=None, colorMap='Greys'):
    ## Sort the data
    x_min, x_max = np.min(X, 0), np.max(X, 0)
    X = (X - x_min) / (x_max - x_min)

    if Species is not None:
        uniqueSpecies = np.unique(Species)
    
    ## Start plotting
    colors = ['r', 'b', 'g', 'o', 'p']
    y_color = []
    y_label = []
    unique_y = np.unique(y)
    for lab in y:
        if lab <5:
            y_label.append('1-4')
            y_color.append(colors[0])
        elif lab < 11:
            y_label.append('5-10')
            y_color.append(colors[1])
        elif lab < 18:
            y_label.append('11-17')
            y_color.append(colors[2])
    y_color = np.asarray(y_color)
    y_label = np.asarray(y_label)
    print(f'y_color shape : {y_color.shape}')
    print(f'y_label shape : {y_label.shape}')
    
    fig = plt.figure(figsize=(5.6,2.8))
    for i in range(len(uniqueSpecies)):
        ax = plt.subplot(1,len(uniqueSpecies),i+1)
        ## Get indicies of the species
        S = []
        L = []
        for j in range(len(unique_y)):
            indicies = np.where(np.logical_and(Species==uniqueSpecies[i], y==unique_y[j]))[0]
            if not list(indicies):
                pass
            else:
                S1 = plt.scatter(X[indicies, 0], X[indicies, 1], c=y_color[indicies], label=y_label[indicies], s=0.1)       
                S.append(S1)
            if not list(y_label[indicies]):
                pass
            else:
                L.append(list(y_label[indicies])[0])

        plt.xticks([]), plt.yticks([])
        if uniqueSpecies[i] =='AA':
            plt.title('An. Arabiensis', fontsize=15)
        elif uniqueSpecies[i] =='AC':
            plt.title('An. Coluzzii', fontsize=15)
        elif uniqueSpecies[i] =='AG':
            plt.title('An. Gambiae', fontsize=15)    

    plt.savefig(saveDir+title+".pdf")
    
    fig = plt.figure(figsize=(5.6,2.8))
    for i in range(len(uniqueSpecies)):
        ax = plt.subplot(1,len(uniqueSpecies),i+1)
        ## Get indicies of the species
        S = []
        L = []
        for j in range(len(unique_y)):
            indicies = np.where(np.logical_and(Species==uniqueSpecies[i], y==unique_y[j]))[0]
            if not list(indicies):
                pass
            else:
                S1 = plt.scatter(X[indicies, 0], X[indicies, 1], c=y_color[indicies], label=y_label[indicies], s=5)       
                S.append(S1)
            if not list(y_label[indicies]):
                pass
            else:
                L.append(list(y_label[indicies])[0])

        plt.xticks([]), plt.yticks([])
        if uniqueSpecies[i] =='AA':
            plt.title('An. Arabiensis', fontsize=15)
        elif uniqueSpecies[i] =='AC':
            plt.title('An. Coluzzii', fontsize=15)
        elif uniqueSpecies[i] =='AG':
            plt.title('An. Gambiae', fontsize=15)

    fig = plt.figure(figsize=(5.6,2.8))
    fig.legend((S), (L), loc='lower center', fontsize=18, markerscale=4.)   

    plt.savefig(saveDir+title+"_legend.pdf", bbox_inches="tight")
num_neighbours = 32
num_min_dist = 0.7
num_metric = 'chebyshev'
## PCA embedding of the digits dataset
dimensions = 2
# print('Shape of X : {}'.format(X.shape))
print("Computing UMAP projection")
t0 = time()
#reducer = umap.UMAP(random_state=42, n_components=dimensions)
reducer = umap.UMAP(random_state=42, n_components=dimensions, n_neighbors=num_neighbours, metric=num_metric, min_dist=num_min_dist)
embedding = reducer.fit(y_predicted)
transformed = reducer.transform(y_predicted)
print('Embedding time: {}'.format(time()-t0))
plot_embedding_country(transformed, y_country, y_species, savedir_main, ('UMAP_Embedded_Mosquito_Country'), colorMap='Set1')

plot_embedding_rearCnd(transformed, y_rearcnd, y_species, savedir_main, ('UMAP_Embedded_Mosquito_RearCnd'), colorMap='Set1')

plot_embedding_age(transformed, y_age, y_species, savedir_main, ('UMAP_Embedded_Mosquito_Age'), colorMap='Set1')<jupyter_output>y_color shape : (40727,)
y_label shape : (40727,)
{'fraction_non_alphanumeric': 0.1412884043607532, 'fraction_numerical': 0.025807730426164518, 'mean_word_length': 3.1007071445988785, 'pattern_counts': {'":': 1, '<': 59, '<?xml version=': 0, '>': 53, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '445109', 'license': 'MIT', 'n_tokens_mistral': 18878, 'n_tokens_neox': 17757, 'n_words': 3257, 'path': 'CNN/CNN-model/MIMI-Paper-CNN-TCValid-only-retrain.ipynb', 'repo_name': 'JoshuaMitton/MIRS-CNN', 'size': '50456'}