<jupyter_start><jupyter_text>Download the image dataset and metadata filesThe dog breed image dataset is made available by Stanford and can by downloaded from [here](http://vision.stanford.edu/aditya86/ImageNetDogs/images.tar)  The dataset can be divided into train and test set by downloading the information files from [here](http://vision.stanford.edu/aditya86/ImageNetDogs/lists.tar)<jupyter_code>#load the information files
full_list = loadmat("data/file_list.mat")
train_list = loadmat("data/train_list.mat")
test_list = loadmat("data/test_list.mat")
#function to convert the information files to a useful format
def create_img_info_df(mat_dict_file):
    image_info_list = []
    for i in range(mat_dict_file['file_list'].shape[0]):
        file_path = mat_dict_file['file_list'][i][0][0]
        breed = re.search(r'(?<=-)\w*', file_path).group()
        image_info_list.append([file_path, breed])
    image_info_df = pd.DataFrame(image_info_list, columns=["file_path", "breed"])
    return image_info_df
full_set_info_df = create_img_info_df(full_list)
train_info_df = create_img_info_df(train_list)
test_info_df = create_img_info_df(test_list)
print("The complete dataset has {} images".format(full_set_info_df.shape[0]))
print("the train set has {} images".format(train_info_df.shape[0]))
print("the test set has {} images".format(test_info_df.shape[0]))
#get a list of unique breeds
breeds = list(full_set_info_df.groupby(by="breed").count().index)
print("Total number of unique breeds: {}".format(len(breeds)))
#create dictionaries to map breed to class_id and vice-versa
breed_to_class_id = {x:y for x, y in zip(breeds, range(len(breeds)))}
class_id_to_breed = {y:x for x,y in breed_to_class_id.items()}
#dump as json files for use during prediction
with open("data/breed_to_class_id.json", "w") as filehandler:
    json.dump(breed_to_class_id, filehandler)
with open("data/class_id_to_breed.json", "w") as filehandler:
    json.dump(class_id_to_breed, filehandler)<jupyter_output><empty_output><jupyter_text>Convert and save the train and test images to hd5 format to make the reading fast during the training process<jupyter_code>import h5py
import numpy as np
import pandas as pd
from PIL import Image
h5_train_file = h5py.File("data/train.h5", 'w')
h5_test_file = h5py.File("data/test.h5", 'w')
#initialize the h5 files with appropriate size (i.e 224 by 224 to be used with resnet) and fill with zeros
train_features = h5_train_file.create_dataset('data', shape=(len(train_info_df), 224, 224, 3), dtype=np.uint8, fillvalue=0)
train_labels = h5_train_file.create_dataset('labels', shape=(len(train_info_df),1), dtype=np.uint8)
test_features = h5_test_file.create_dataset('data', shape=(len(test_info_df), 224, 224, 3), dtype=np.uint8, fillvalue=0)
test_labels = h5_test_file.create_dataset('labels', shape=(len(test_info_df),1), dtype=np.uint8)
#function to convert all the images to a defined width and height
def resizeImage(size, image):
    image.thumbnail(size, Image.ANTIALIAS)
    background = Image.new('RGB', size, (0, 0, 0))
    background.paste(image, (int((size[0] - image.size[0]) / 2), int((size[1] - image.size[1]) / 2)))
    return background
# write image and label to h5_train_file and h5_test_file
for i in range(len(train_info_df)):
    image_file_name = train_info_df.iloc[i]['file_path']
    img = Image.open("data/Images/" + image_file_name)
    img = resizeImage((224,224), img)
    img = np.array(img)
    train_features[i] = img #write to h5 dataset
    train_labels[i] = breed_to_class_id[train_info_df.iloc[i]['breed']] #write to h5 dataset
    
for i in range(len(test_info_df)):
    image_file_name = test_info_df.iloc[i]['file_path']
    img = Image.open("data/Images/" + image_file_name)
    img = resizeImage((224,224), img)
    img = np.array(img)
    test_features[i] = img #write to h5 dataset
    test_labels[i] = breed_to_class_id[test_info_df.iloc[i]['breed']] #write to h5 dataset
h5_train_file.close()
h5_test_file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12693266832917705, 'fraction_numerical': 0.0199501246882793, 'mean_word_length': 5.903614457831325, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '172325', 'license': 'MIT', 'n_tokens_mistral': 1457, 'n_tokens_neox': 1410, 'n_words': 334, 'path': 'training/data_preperation.ipynb', 'repo_name': 'brianpinto91/dog-breed-prediction-gcloud', 'size': '4010'}