<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive/')
cd /content/drive/My Drive/Make/face2bmi-keras/
# -*- coding: utf-8-unix -*-

import os, sys
sys.path.append("/face2bmi-keras/keras-vggface")

import glob
import pandas as pd
import numpy as np
import h5py
import re

# Keras: Tensorflow backend
from keras.preprocessing.image import load_img, img_to_array, array_to_img
from keras.utils import np_utils


def numericalSort(value):
    numbers = re.compile(r'(\d+)')
    parts = numbers.split(value)
    parts[1::2] = map(int, parts[1::2])
    return parts

def flush_progress_bar(barname, rate):
    number = int(rate/0.05)+1
    bar = ('#' * number).ljust(20, '-')
    
    sys.stdout.write(f"\r{barname} : [{bar}] {rate*100:.2f}%")

class BMIDataset:
    def __init__(self):
        self.image_shape = (224, 224, 3)
        self.dataset_store_file_path = "./stored_bmi_dataset/bmi_dataset.h5"
        
    def load_data(self, image_dir_path):
        if os.path.exists(self.dataset_store_file_path):
            print("Stored dataset found!\nLoading...")
            
            with h5py.File(self.dataset_store_file_path, 'r') as loaded_dataset:
                x_train = loaded_dataset["x_train"].value
                t_train = loaded_dataset["t_train"].value
                x_test = loaded_dataset["x_test"].value
                t_test = loaded_dataset["t_test"].value


        else:
            print("Stored dataset not found.\nMaking...")
            x = []
            t = []
        
            files = sorted(glob.glob(image_dir_path + "/*.bmp"), key=numericalSort)
            
            n = len(files)
            i = 0
            
            # Get image and bmi value from original dataset
            for imgfile in files:
                img = load_img(imgfile, target_size=(224,224) ) #(self.image_shape[:2]))
                img_array = img_to_array(img)
                filename = os.path.basename(imgfile)
                #bmi = self.filename2bmi(filename)
                bmiLevel = self.filename2bmiLevel(filename)
                
                flush_progress_bar("Loading image", i/n)
                
                x.append(img_array)
                t.append(bmiLevel)
        
                i += 1
        
            x = np.array(x)
            x = x.astype("float32") / 255.0
            t = np.array(t)
            t = t.astype("int32")
            t_categorical = np_utils.to_categorical(t, 9)

            x_train = x[:3368]
            x_test = x[3368:]
            t_train = t_categorical[:3368]
            t_test = t_categorical[3368:]
            
            # Store dataset to file
            if not os.path.exists(os.path.dirname(self.dataset_store_file_path)):
                os.mkdir(os.path.dirname(self.dataset_store_file_path))
            
            with h5py.File(self.dataset_store_file_path, "w") as store_dataset:
                store_dataset.create_dataset("x_train", data=x_train)
                store_dataset.create_dataset("t_train", data=t_train)
                store_dataset.create_dataset("x_test", data=x_test)
                store_dataset.create_dataset("t_test", data=t_test)
                store_dataset.flush()
    
        return((x_train, t_train), (x_test, t_test))

    def filename2bmiLevel(self, filename):    
        # Get index
        tempstr = filename.replace(".bmp", "")
        tempstrs = tempstr.split("_")
        index = int(tempstrs[1])

        # Get BMI from csv file by index
        df = pd.read_csv("./DataFromMIT-CSAIL/data.csv")
        
        bmi = df["bmi"][index]

        bmiLevel = 1

        if bmi < 16.0:
            bmiLevel = 8
        elif bmi >= 16.0 and bmi < 17.0:
            bmiLevel = 7
        elif bmi >= 17.0 and bmi < 18.5:
            bmiLevel = 6
        elif bmi >= 18.5 and bmi < 25.0:
            bmiLevel = 5
        elif bmi >= 25.0 and bmi < 30.0:
            bmiLevel = 4
        elif bmi >= 30.0 and bmi < 35.0:
            bmiLevel = 3
        elif bmi >= 35.0 and bmi < 40.0:
            bmiLevel = 2
        elif bmi >= 40.0:
            bmiLevel = 1
        else:
            bmiLevel = 5
            
        return bmiLevel
    
    
    def filename2bmi(self, filename):    
        # Get index
        tempstr = filename.replace(".bmp", "")
        tempstrs = tempstr.split("_")
        index = int(tempstrs[1])

        # Get BMI from csv file by index
        df = pd.read_csv("./DataFromMIT-CSAIL/data.csv")
        
        bmi = df["bmi"][index]
        
        return float(bmi)



def main():
    
    # Load dataset
    dataset = BMIDataset()
    ( (x_train, t_train), (x_test, t_test) ) = dataset.load_data("./DataFromMIT-CSAIL/Images")

    print('\n')
    
    print(t_test[:10])
    
    

if __name__ == "__main__":
    main()

# -*- coding: utf-8-unix -*-

import os, sys
sys.path.append("./keras-vggface")

import glob
import pandas as pd
import numpy as np
import h5py

from keras.preprocessing.image import load_img, img_to_array, array_to_img
from keras.engine import  Model
from keras.layers import Flatten, Dense, Input, Dropout
from keras.layers.normalization import BatchNormalization
from keras.optimizers import SGD
from keras_vggface.vggface import VGGFace

from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import StratifiedKFold
from keras.models import load_model



def network():
    #custom parameters
    nb_class = 9
    hidden_dim = 512
    learning_late = 0.01

    vgg_model = VGGFace(include_top=False, input_shape=(224, 224, 3))
    last_layer = vgg_model.get_layer('pool5').output
    x = Flatten(name='flatten')(last_layer)
    x = Dense(hidden_dim, activation='relu', name='fc6')(x)
    x = Dense(hidden_dim, activation='relu', name='fc7')(x)
    out = Dense(nb_class, activation='softmax', name='fc8')(x)
    custom_vgg_model = Model(vgg_model.input, out)
    vgg_model.summary()
    optimizer = SGD(lr=learning_late)
    custom_vgg_model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["mae"])

    return custom_vgg_model


def main():
    # Parameters
    batch_size = 50
    epochs = 400
    verbose = 0 #1
    validation_split = 0.2
    
    # Load dataset
    dataset = BMIDataset()
    (x_train, t_train), (x_test, t_test) = dataset.load_data("./Data/Images")
    input_shape = dataset.image_shape
    
    model = network()    
    
    
    print(t_test)

    
    # Learn
    model.fit(x_train, t_train,
              batch_size=batch_size, epochs=epochs, callbacks=None, 
              verbose=verbose, validation_split=validation_split)
    model.save('saved_model.h5')
    
    # # Eval
    score = model.evaluate(x_test, t_test, verbose=verbose, batch_size=batch_size)
    print("\nTest loss:", score[0])
    print('Test mean abs error:', score[1])
    
    

if __name__ == "__main__":
    main()

from keras.models import load_model

batch_size = 50
verbose = 1 #1
validation_split = 0.2
    
# Load dataset
dataset = BMIDataset()
(x_train, t_train), (x_test, t_test) = dataset.load_data("./Data/Images")
input_shape = dataset.image_shape

model = load_model('saved_model.h5')
score = model.evaluate(x_test, t_test, verbose=verbose, batch_size=batch_size)
print("\nTest loss:", score[0])
print('Test mean abs error:', score[1])<jupyter_output>Stored dataset found!
Loading...
{'fraction_non_alphanumeric': 0.116, 'fraction_numerical': 0.021733333333333334, 'mean_word_length': 2.2598870056497176, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '160172', 'license': 'MIT', 'n_tokens_mistral': 2625, 'n_tokens_neox': 2475, 'n_words': 515, 'path': 'face2bmi.ipynb', 'repo_name': 'kooga0682/FAIRCUT-Trainer', 'size': '7500'}