<jupyter_start><jupyter_text>Create pickle files from helen dataset images with pandas dataframes. Imports.<jupyter_code>import matplotlib.pyplot as plt
from PIL import Image
import io

import numpy as np
import pandas as pd
import tensorflow as tf

import time
import os
import sys<jupyter_output><empty_output><jupyter_text>Setup folder path for reading images.<jupyter_code>path = '/content/drive/MyDrive/Colab Notebooks/facial_landmark_pred_example/'
sys.path.append(path)
# setup folder path

data_folder = path + "helen_data/"
print(data_folder)

data_sub_folders = os.listdir(data_folder)
label_path = data_sub_folders[0] + "/" + data_sub_folders[0] + "/"
print(label_path)

# Show files list
files = os.listdir(data_folder + label_path)
print(len(files))
print(files)<jupyter_output><empty_output><jupyter_text>Read images<jupyter_code>def read_img(file):
    """ 
    Read a single image file
    """
    split_line = file.split("/")
    split_line = split_line[-1].replace(".jpg", "")
    img_name = split_line

    with open(file, 'rb') as img_file:
        img = img_file.read()

    return img_name, img    
def read_imgs(folder_path):
    """ 
    Read image files from a single folder.
    """
    files = os.listdir(folder_path)
    imgs_name = []
    imgs = []
    for f in files:
        img_name, img = read_img(folder_path + f)
        imgs_name.append(img_name)
        imgs.append(img)                
    
    return imgs_name, imgs
def read_imgs_folder(folder_list, folder_num):
    """
    Read image files from multiple folders.
    """
    imgs_name_list = []
    imgs_list = []
    for i, data_sub_folders in enumerate(folder_list):
        if i>0:
            folder_path = data_folder + data_sub_folders + "/" + data_sub_folders + "/"
            imgs_name, imgs = read_imgs(folder_path)
            imgs_name_list.append(imgs_name)
            imgs_list.append(imgs) 

            # Write to pandas dataframe.
            names_df = pd.DataFrame(imgs_name)
            names_df.columns = ['file_name']    
            imgs_df = pd.DataFrame(imgs)
            imgs_df.columns = ['img']

            # Concat Names & images df.
            names_p_imgs_df = pd.concat([names_df, imgs_df], axis=1)     
            names_p_imgs_df.to_pickle(path + "helen_pickle/" + "helen_ds_img_" + str(i) + ".pickle")
            print(names_p_imgs_df.head())

    return imgs_name_list, imgs_list<jupyter_output><empty_output><jupyter_text>Read images & create pickle files.<jupyter_code>folder_num = 5      # 1 to 5

start_time = time.time()
imgs_name_list, imgs_list = read_imgs_folder(data_sub_folders, folder_num)
print("--- %s seconds ---" % (time.time() - start_time))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1092313350496506, 'fraction_numerical': 0.002942258183155572, 'mean_word_length': 3.1910631741140216, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '395450', 'license': 'MIT', 'n_tokens_mistral': 966, 'n_tokens_neox': 945, 'n_words': 191, 'path': 'tfrec_gen/helen_ds_img_pds.ipynb', 'repo_name': 'ChuaCheowHuan/facial_landmark_pred_example', 'size': '2719'}