<jupyter_start><jupyter_text>Drowsiness Detection: Quantitative AnalysisAuthors: <NAME> and <NAME><jupyter_code>from google.colab import drive
drive.mount('/content/drive/')

import sys
sys.path.insert(0,'/content/drive/MyDrive/DrowsinessDetection/source')
import os
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

import torch
import torchvision
import torchvision.transforms as transforms
import torchvision.datasets as datasets

import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import tools
import tests
import pandas as pd
# When running on your own computer, you can specify the data directory by:
data_dir = tools.select_data_dir('/content/drive/MyDrive/DrowsinessDetection')
source_dir = '/content/drive/MyDrive/DrowsinessDetection/source'
print(data_dir)
# Select the device for training (use GPU if you have one)
device = torch.device('cuda:0')
#device = torch.device('cpu')
data_dir_yawn = os.path.join(data_dir,'data/yawn_data')
data_dir_eyes = os.path.join(data_dir,'data/eyes_data')
data_dir_fatigue = os.path.join(data_dir,'data/fatigue_data')<jupyter_output><empty_output><jupyter_text>Independent/Dependent Variables and Expected PropertiesOur data are three labelled image datasets intended to use for deep learning model. There are two variables in our data: Images and Labels. The data is of categorical nature that means that both variables are belonging to a category, in our case we call the categories classes. The goal of this quantitative analysis to explore the distribution of the frequency of the images among the categories and datasets. This will help us to find where bias could be introduced because of the distribution among the classes and datasets as well as because of the size of the datasets. Independent variables:  * Images* DatasetsExpected properties:    *   Image data in color*   Different image sizes*   Evenly distributed among classes and datasetsDependent variables:    *   Labels   Datasets PreprocessingPreprocessing consists of:1.   Resizing the images2.   Turn it into grayscale colors3.   Convert into a tensor4.   Normalize the tensor values<jupyter_code>data_transform = transforms.Compose([
        transforms.Resize((28,28)),     #Questionable?
        transforms.Grayscale(),               
        transforms.ToTensor(),
        transforms.Normalize((0.5,), (0.5,))  # Scale images to [-1, 1]
])<jupyter_output><empty_output><jupyter_text>Yawn Dataset<jupyter_code>dataset_yawn = datasets.ImageFolder(data_dir_yawn, transform=data_transform)

size_yawn = len(dataset_yawn)
print(size_yawn)
trainset_yawn, testset_yawn = torch.utils.data.random_split(dataset_yawn, [len(dataset_yawn)-150, 150])

classes = ['yawn', 'no_yawn']

trainloader_yawn = torch.utils.data.DataLoader(trainset_yawn, batch_size=32, shuffle=True)
testloader_yawn = torch.utils.data.DataLoader(testset_yawn, batch_size=5, shuffle=False)

train_yawn = len(trainset_yawn)
test_yawn = len(testset_yawn)
print(train_yawn)
print(test_yawn)<jupyter_output>1450
1300
150<jupyter_text>Eyes Dataset<jupyter_code>dataset_eyes = datasets.ImageFolder(data_dir_eyes, transform=data_transform)

size_eyes = len(dataset_eyes)
print(size_eyes)
trainset_eyes, testset_eyes = torch.utils.data.random_split(dataset_eyes, [len(dataset_eyes)-240, 240])

classes = ['closed_eyes', 'open_eyes']

trainloader_eyes = torch.utils.data.DataLoader(trainset_eyes, batch_size=32, shuffle=True)
testloader_eyes = torch.utils.data.DataLoader(testset_eyes, batch_size=5, shuffle=False)

train_eyes = len(trainset_eyes)
test_eyes = len(testset_eyes)
print(train_eyes)
print(test_eyes)<jupyter_output>2423
2183
240<jupyter_text>Fatigue Dataset<jupyter_code>dataset_fatigue = datasets.ImageFolder(data_dir_fatigue, transform=data_transform)

size_fatigue = len(dataset_fatigue)
print(size_fatigue)
trainset_fatigue, testset_fatigue = torch.utils.data.random_split(dataset_fatigue, [len(dataset_fatigue)-390, 390])

classes = ['alert', 'non_vigilant', 'tired']

trainloader_fatigue = torch.utils.data.DataLoader(trainset_fatigue, batch_size=32, shuffle=True)
testloader_fatigue = torch.utils.data.DataLoader(testset_fatigue, batch_size=5, shuffle=False)

train_fatigue = len(trainset_fatigue)
test_fatigue = len(testset_fatigue)
print(train_fatigue)
print(test_fatigue)<jupyter_output>3907
3517
390<jupyter_text>Visualizing the data Show Example Images of Yawn Dataset<jupyter_code>#Plot a few images in greyscale
images, labels = iter(trainloader_yawn).next()
tests.plot_images(images[:8], n_rows=2)
print(images[0].shape)<jupyter_output>torch.Size([1, 28, 28])<jupyter_text>Show Example Images of Eyes Dataset<jupyter_code>#Plot a few images in greyscale
images, labels = iter(trainloader_eyes).next()
tests.plot_images(images[:8], n_rows=2)
print(images[0].shape)<jupyter_output>torch.Size([1, 28, 28])<jupyter_text>Show Example Images of Fatigue Dataset<jupyter_code>#Plot a few images in greyscale
images, labels = iter(trainloader_fatigue).next()
tests.plot_images(images[:8], n_rows=2)
print(images[0].shape)<jupyter_output>torch.Size([1, 28, 28])<jupyter_text>Comparison of Dataset Size to Find Size Bias<jupyter_code>plotdata = pd.DataFrame(
    {"Dataset Size": [size_yawn, size_eyes, size_fatigue]}, 
    index=["Yawn", "Eyes", "Fatigue"])
plt.style.use('fivethirtyeight') 
plotdata.plot(kind="bar")
plt.title("Comparison of Dataset Size")
plt.xlabel("Number of Images")
plt.ylabel("Datasets")
plotdata = pd.DataFrame(
    {"Total": [size_yawn, size_eyes, size_fatigue],
     "Trainset":[train_yawn, train_eyes, train_fatigue],
     "Testset":[test_yawn, test_eyes, test_fatigue]
     }, 
    index=["Yawn", "Eyes", "Fatigue"]
)
plotdata.head()
#plotdata.apply(lambda x: x*100/sum(x), axis=1)
stacked_data = pd.DataFrame(
    {
     "Trainset":[train_yawn, train_eyes, train_fatigue],
     "Testset":[test_yawn, test_eyes, test_fatigue]
     }, 
    index=["Yawn", "Eyes", "Fatigue"]
)
plt.style.use('fivethirtyeight') 
stacked_data.plot(kind="bar", stacked=True)
plt.title("Partition of Datasets")
plt.xlabel("Datasets")
plt.ylabel("Number of Images")
from collections import Counter
import matplotlib.patches as mpatches
counts_yawn = Counter(x[1] for x in dataset_yawn.imgs)
counts_eyes = Counter(x[1] for x in dataset_eyes.imgs)
counts_fatigue = Counter(x[1] for x in dataset_fatigue.imgs)

classes_data = pd.DataFrame(
    {
     "Class Size":[counts_yawn[0], counts_yawn[1], counts_eyes[0],counts_eyes[1],counts_fatigue[0],counts_fatigue[1],counts_fatigue[2]],
     }, 
    index=['no_yawn', 'yawn', 'closed_eyes', 'open_eyes', 'alert', 'non_vigilant','tired']
)
plt.style.use('fivethirtyeight') 
classes_data['Class Size'].plot(kind="bar", stacked=True, color=['red', 'red', 'green', 'green','blue', 'blue', 'blue'])
plt.title("Partition of Datasets among Classes")
plt.xlabel("Classes")
plt.ylabel("Number of Images")
y = mpatches.Patch(color='red', label= 'yawn dataset')
e = mpatches.Patch(color='green', label= 'eyes dataset')
f = mpatches.Patch(color='blue', label= 'fatigue dataset')
plt.legend(handles = [y,e,f], fontsize = 'x-small')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11897421524663677, 'fraction_numerical': 0.015975336322869956, 'mean_word_length': 6.817086527929901, 'pattern_counts': {'":': 7, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '185939', 'license': 'MIT', 'n_tokens_mistral': 2600, 'n_tokens_neox': 2457, 'n_words': 543, 'path': 'source/quantitative_analysis.ipynb', 'repo_name': 'GasserElAzab/ResNet-Model-For-Generalized-Driver-Drowsiness-Detection', 'size': '7149'}