<jupyter_start><jupyter_text>Writing Tiny ImageNet dataset into HDF5 file  Importing Libraries<jupyter_code>from config import tiny_imagenet_config as config
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from compvis.datasets import HD5FDatasetWriter
from imutils import paths
import numpy as np
import json
import cv2
import os
# grab the paths to the training images, then extract the training class labels and encode them
trainPaths = list(paths.list_images(config.TRAIN_IMAGES))
trainLabels = [p.split(os.path.sep)[-3] for p in trainPaths]
le = LabelEncoder()
trainLabels = le.fit_transform(trainLabels)
# Splitting the dataset into training and validation set

split = train_test_split(trainPaths, trainLabels, test_size=config.NUM_TEST_IMAGES,
                         stratify=trainLabels, random_state=42)
(trainPaths, testPaths, trainLabels, testLabels) = split
# load the validation filename => class from file and then use these mappings to build the validation paths and label lists
M = open(config.VAL_MAPPINGS).read().strip().split("\n")
M = [r.split("\t")[:2] for r in M]
valPaths = [os.path.sep.join([config.VAL_IMAGES, m[0]]) for m in M]
valLabels = le.transform([m[1] for m in M])

# construct a list pairing the training, validation, and testing image paths along with their corresponding labels and output HDF5
# files

datasets = [("train", trainPaths, trainLabels, config.TRAIN_HDF5),
            ("val", valPaths, valLabels, config.VAL_HDF5),
            ("test", testPaths, testLabels, config.TEST_HDF5)]
# initialize the lists of RGB channel averages
(R, G, B) = ([], [], [])
# loop over the dataset tuples
for (dTypes, paths, labels, outputPath) in datasets:
    # Creating the HDF5 writter
    print("[INFO] building {}...".format(outputPath))
    writer = HD5FDatasetWriter((len(paths), 64, 64, 3), outputPath)
    # loop over the image paths
    for (i, (path, label)) in enumerate(zip(paths, labels)):
        # Loading image from disk
        image = cv2.imread(path)
        # if we are building the training dataset, then compute the
        # mean of each channel in the image, then update the
        # respective lists
        if dTypes == "train":
            (b, g, r) = cv2.mean(image)[:3]
            R.append(r)
            G.append(g)
            B.append(b)
        writer.add([image], [label])
    
    # closing the current dataset
    writer.close()
# construct a dictionary of averages, then serialize the means to a JSON file
print("[INFO] serializing means...")
D = {"R": np.mean(R), "G": np.mean(G), "B": np.mean(B)}
f = open(config.DATASET_MEAN, "w")
f.write(json.dumps(D))
f.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11147902869757174, 'fraction_numerical': 0.008462104488594555, 'mean_word_length': 3.8990990990990992, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '268653', 'license': 'Apache-2.0', 'n_tokens_mistral': 834, 'n_tokens_neox': 821, 'n_words': 281, 'path': 'Practical Examples/11 - GoogLeNet/buildind_tiny_dataset.ipynb', 'repo_name': 'IgorMeloS/Computer-Vision-Training', 'size': '2718'}