<jupyter_start><jupyter_text>Params<jupyter_code>MODEL_SAVE = 'service_type_model.h5'
FAST_RUN = False
IMAGE_WIDTH = 300
IMAGE_HEIGHT = 300
IMAGE_SIZE = (IMAGE_WIDTH, IMAGE_HEIGHT)
IMAGE_CHANNELS = 1
IMG_DIR = 'imagens/'
BATCH_SIZE = 32
NUM_CLASSES = 4
## Start by connecting gdrive into the google colab
from google.colab import drive
drive.mount('/content/gdrive')
! cp "/content/gdrive/My Drive/Projects/MaskDetection/dataset/mask_dataset_images.zip" .
! unzip mask_dataset_images.zip<jupyter_output><empty_output><jupyter_text>Create dataset<jupyter_code>dataset = []

for fold in os.listdir(IMG_DIR):
    for filename in os.listdir(f'{IMG_DIR}/{fold}'):
        dataset.append((f'{fold}/{filename}', fold))

df = pd.DataFrame(dataset, columns=['filename', 'category'])
df_train, df_test = train_test_split(df, random_state=42, stratify=df.category, test_size=.2)
df_train['set'] = 'train'
df_test['set'] = 'test'
df = df_train.append(df_test)
df.to_csv('dataset.csv', index=False)
df.head()<jupyter_output><empty_output><jupyter_text>Read and prepare data<jupyter_code>df = pd.read_csv('dataset.csv')
df.head()
train_df = df[df.set == 'train'].reset_index(drop=True)
validate_df = df[df.set == 'test'].reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Build Model<jupyter_code>base_model = MobileNet(
    weights= None, 
    include_top=False, 
    input_shape= (IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS)
)

x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(256,activation='relu')(x) 
x = Dropout(0.2)(x)
predictions = Dense(NUM_CLASSES, activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=predictions)

opt = Adam(lr=0.000125)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>Callbacks<jupyter_code>callbacks_list = [
    ModelCheckpoint('/content/gdrive/My Drive/Projects/MaskDetection/weights/service_weights.h5', 
                    monitor='val_accuracy', verbose=1, save_best_only=True, mode='max'),
    EarlyStopping(monitor='val_accuracy', patience=5),
    ReduceLROnPlateau(monitor='val_accuracy', 
                                            patience=3, 
                                            verbose=1, 
                                            factor=0.5, 
                                            min_lr=0.00001)
]<jupyter_output><empty_output><jupyter_text>Generators<jupyter_code>def add_noise(img):
    '''Add random noise to an image'''
    VARIABILITY = 8
    deviation = VARIABILITY*random.random()
    noise = np.random.normal(0, deviation, img.shape)
    img += noise
    np.clip(img, 0., 255.)
    return img


train_datagen = ImageDataGenerator(
    brightness_range=[0.2, 1.6],
    rescale=1. / 255,
    rotation_range=0, 
    width_shift_range=0.1,
    height_shift_range=0.1, 
    shear_range=0.2, 
    zoom_range=0.2,
    horizontal_flip=True, 
    fill_mode="nearest",
    preprocessing_function=add_noise,
)
train_generator = train_datagen.flow_from_dataframe(
    train_df, 
    IMG_DIR, 
    x_col='filename',
    y_col='category',
    target_size=IMAGE_SIZE,
    color_mode = 'grayscale',
    class_mode='categorical',
    batch_size=BATCH_SIZE
)
validation_datagen = ImageDataGenerator(rescale=1./255)
validation_generator = validation_datagen.flow_from_dataframe(
    validate_df, 
    IMG_DIR,
    x_col='filename',
    y_col='category',
    target_size=IMAGE_SIZE,
    color_mode = 'grayscale',
    class_mode='categorical',
    shuffle=False,
    batch_size=BATCH_SIZE
)<jupyter_output><empty_output><jupyter_text>Fit model<jupyter_code>total_train = train_df.shape[0]
total_validate = validate_df.shape[0]
model.load_weights('/content/gdrive/My Drive/Projects/MaskDetection/weights/service_weights.h5')
epochs=50 if FAST_RUN else 50

history = model.fit_generator(
    train_generator, 
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps=total_validate//BATCH_SIZE,
    steps_per_epoch=total_train//BATCH_SIZE,
    callbacks=callbacks_list
)
nb_samples = validate_df.shape[0]
predict = model.predict_generator(validation_generator, steps=np.ceil(nb_samples/BATCH_SIZE))
validate_df['pred'] = np.argmax(predict, axis=-1)
label_map = dict((v,k) for k,v in train_generator.class_indices.items())
validate_df['pred'] = validate_df['pred'].replace(label_map)
label_map

from sklearn.metrics import classification_report, confusion_matrix
print(classification_report(validate_df.category, validate_df.pred))
print(confusion_matrix(validate_df.category, validate_df.pred))
model.save('model_colab.h5')
model.save('/content/gdrive/My Drive/Projects/MaskDetection/model.h5')
import numpy as np
import cv2 as cv
import pandas as pd
import os
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import confusion_matrix

def load_dataframe():
    '''
    Carrega um dataframe Pandas com as imagens para o treinamento do modelo
    '''
    data_info = {
        "filename": [],
        "label": [],
        "target": [],
        "image": [],
    }

    with_mask = os.listdir("imagens/maskon")
    without_mask = os.listdir("imagens/maskoff")
    mask_chin = os.listdir("imagens/maskchin")
    mask_mouth = os.listdir("imagens/maskmouth")

    for filename in with_mask:
        data_info["filename"].append(f"imagens/maskchin/{filename}")
        data_info["label"].append(f"Mask only in the chin")
        data_info["target"].append(1)
        img = cv.imread(f"imagens/maskchin/{filename}")
        img = cv.cvtColor(img, cv.COLOR_BGR2GRAY).flatten()
        data_info["image"].append(img)

    for filename in with_mask:
        data_info["filename"].append(f"imagens/maskmouth/{filename}")
        data_info["label"].append(f"Mask below the nose")
        data_info["target"].append(1)
        img = cv.cvtColor(cv.imread(f"imagens/maskmouth/{filename}"), cv.COLOR_BGR2GRAY).flatten()
        data_info["image"].append(img)

    for filename in without_mask:
        data_info["filename"].append(f"imagens/maskoff/{filename}")
        data_info["label"].append(f"Without Mask")
        data_info["target"].append(2)
        img = cv.cvtColor(cv.imread(f"imagens/maskoff/{filename}"), cv.COLOR_BGR2GRAY).flatten()
        data_info["image"].append(img)

    for filename in with_mask:
        data_info["filename"].append(f"imagens/maskon/{filename}")
        data_info["label"].append(f"With Mask")
        data_info["target"].append(3)
        img = cv.cvtColor(cv.imread(f"imagens/maskon/{filename}"), cv.COLOR_BGR2GRAY).flatten()
        data_info["image"].append(img)

        
    dataframe = pd.DataFrame(dados)

    return dataframe
def train_test(dataframe):
    '''
    Divide o dataframe em conjunto de treino e teste
    '''
    X = list(dataframe["image"])
    y = list(dataframe["target"])

    return train_test_split(X, y, train_size=0.40, random_state=13)


def pca_model(X_train):
    '''
    PCA para extração de features das imagens
    '''
    pca = PCA(n_components=50)
    pca.fit(X_train)
    
    return pca
dataframe = load_dataframe() #Carregando dataframe com as imagens para treinamento

X_train, X_test, y_train, y_test = functions.train_test(dataframe) #Dividindo conjuntos de treino e teste
pca = functions.pca_model(X_train) #Modelo PCA para extração de features da imagem<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1265568501406187, 'fraction_numerical': 0.012588723717691175, 'mean_word_length': 4.251758087201125, 'pattern_counts': {'":': 4, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '322708', 'license': 'MIT', 'n_tokens_mistral': 2625, 'n_tokens_neox': 2538, 'n_words': 411, 'path': 'train_keras_model_mobilenet.ipynb', 'repo_name': 'aslamkhan1998/face_mask_detection', 'size': '7467'}