<jupyter_start><jupyter_text>Get LabelsCode to build dataframes that contain directory listings of train, test and validate extracted JPEG frames including labels  Code to build dataframes that contain directory listings of train, test and validate extracted avi videos including labels  Assumed working from DAiSEE dir<jupyter_code>import os
import shutil
import glob
from pathlib import Path
import pandas as pd
import numpy as np
os.getcwd()
fps = '2FPS'
frame_dir = '../../data/DAiSEE/' + fps + '/dataImages/'
label_path = '../../data/DAiSEE/Labels/'
out_dir = '../../data/DAiSEE/' + fps + '/data/' 

usage = ['Train', 'Test', 'Validation']
def get_labels (frame_dir, usage):
    df_l = pd.read_csv(label_path + usage + 'Labels.csv')
    df_l['basename'] = df_l['ClipID'].str[:-4]
    
    # Get Data Files
    df_j = pd.DataFrame([file_path for file_path in Path(frame_dir + usage).glob('*.jpg')], columns=['file'])
    df_j["root"] = df_j["file"].apply(lambda x: os.path.split(os.path.split(x)[0])[1])
    df_j['basefile'] = df_j['file'].apply(lambda x: os.path.basename(x))
    df_j['sequence'] = df_j['basefile'].apply(lambda x: int(x[x.find('_')+1:-4]))
    df_j['basename'] = df_j['basefile'].apply(lambda x: x[:x.find('_')])  
    
    # Merge and cleanup
    df = pd.merge(df_j, df_l, on='basename', how='inner')
    df = pd.merge(df_j, df_l, on='basename', how='inner')
    df = pd.merge(df_j, df_l, on='basename', how='inner')  
        
    df.rename(columns={'Frustration ':'Frustration'}, inplace = True)   
    df['file'] = df['file'].apply(lambda x: str(x))

    return df
df_train = get_labels(frame_dir, 'Train')
df_test = get_labels(frame_dir, 'Test')
df_val = get_labels(frame_dir, 'Validation')
df_val['Boredom'].value_counts()
# write dataframe to pickle in case we need it later
df_train.to_pickle(frame_dir + "/df_train.pkl")
df_test.to_pickle(frame_dir + "/df_test.pkl")
df_val.to_pickle(frame_dir + "/df_val.pkl")
print(df_train.shape)
print(df_test.shape)
print(df_val.shape)
def save_arrays(df, usage):
    filepath = df['file'].to_numpy()
    label = np.array(df[['Boredom', 'Engagement', 'Confusion', 'Frustration']]) 

    np.save(f"{str(frame_dir)}/x_{usage.lower()}", filepath, allow_pickle=True)
    np.save(f"{str(frame_dir)}/y_{usage.lower()}", label)
save_arrays(df_train, 'train')
save_arrays(df_test, 'test')
save_arrays(df_val, 'validation')<jupyter_output><empty_output><jupyter_text>Move files to appropriate directories (train, test, val) Create Class Subdirectories for ease of use with Tenrorflow datasetsInititally treat as a binary problem (cannot be a bit engaged and a bit bored  Therefore create a binary class column based on boredom  If there are duplicated we are going to keep going, because one less image file should not make a difference.<jupyter_code>def class_encoder(df):
    y = pd.get_dummies(df['Boredom'], prefix='b')
    df = pd.concat([df,y], axis = 1)
    df.rename(columns={"b_0": "b0", "b_1": "b1", "b_2": "b2", "b_3": "b3"}, inplace = True)
    
    return df
df_train = class_encoder(df_train)
df_test = class_encoder(df_test)
df_val = class_encoder(df_val)
# Check values for 'Boredom'
df_val.groupby(['Boredom']).size()
# Simple function to copy images to correct file structure
# As we have some duplicate file names, instead of fixing we will ignore as we have more than enough images
def copy_files(source, destination):
    for f in source:
        destination_file = os.path.join(destination, os.path.basename(f))
        shutil.copy(os.fspath(f), destination_file)
# Create image file structure for bored/not bored and copy files
cols = ['b0', 'b1', 'b2', 'b3']
dirs = ['train', 'test', 'validation']

for d in dirs:
    for c in cols:
        data_dir = out_dir + '/' + d + '/' + c
        if not os.path.exists(data_dir):
            os.makedirs(data_dir)
print(out_dir)
# Move Train Images
copy_files(df_train[df_train['b0']==1]['file'].to_list(), out_dir + 'train/b0/')
copy_files(df_train[df_train['b1']==1]['file'].to_list(), out_dir + 'train/b1/')
copy_files(df_train[df_train['b2']==1]['file'].to_list(), out_dir + 'train/b2/')
copy_files(df_train[df_train['b3']==1]['file'].to_list(), out_dir + 'train/b3/')
# Move Test Images
copy_files(df_test[df_test['b0']==1]['file'].to_list(), out_dir + '/test/b0/')
copy_files(df_test[df_test['b1']==1]['file'].to_list(), out_dir + '/test/b1/')
copy_files(df_test[df_test['b2']==1]['file'].to_list(), out_dir + '/test/b2/')
copy_files(df_test[df_test['b3']==1]['file'].to_list(), out_dir + '/test/b3/')
# Move Validation Images
copy_files(df_val[df_val['b0']==1]['file'].to_list(), out_dir + '/validation/b0/')
copy_files(df_val[df_val['b1']==1]['file'].to_list(), out_dir + '/validation/b1/')
copy_files(df_val[df_val['b2']==1]['file'].to_list(), out_dir + '/validation/b2/')
copy_files(df_val[df_val['b3']==1]['file'].to_list(), out_dir + '/validation/b3/')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17764227642276423, 'fraction_numerical': 0.011178861788617886, 'mean_word_length': 5.166666666666667, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '234524', 'license': 'MIT', 'n_tokens_mistral': 1828, 'n_tokens_neox': 1812, 'n_words': 405, 'path': 'src/cnn/.ipynb_checkpoints/1_get_labels-checkpoint.ipynb', 'repo_name': 'SextonCJ/w251Project', 'size': '4920'}