<jupyter_start><jupyter_text>Fashiongen - subset<jupyter_code>import h5py
import numpy as np

import matplotlib.pyplot as plt

from PIL import Image<jupyter_output><empty_output><jupyter_text>---<jupyter_code>dataset_h5_filename = '/media/ai/backup/datasets/fashiongen/fashiongen_256_256_train.h5'

# open the file
file_h5 = h5py.File(dataset_h5_filename, mode='r')
print('number of samples = %d' % (len(file_h5['input_description'])))

print 'Caption of the first sample:'
desc = str(file_h5['input_description'][0])
print(desc)

index = desc.find('.')
if index is not -1:
    desc = desc[:index]
print('only first sentence: %s' % (desc))
print('caterory: %s' % (file_h5['input_category'][0]))
print type(file_h5['input_category'][0])

image = Image.fromarray(file_h5['input_image'][0].astype('uint8'), 'RGB')
plt.imshow(image)<jupyter_output>caterory: ['SHIRTS']
<type 'numpy.ndarray'><jupyter_text>---  Make a subset of 8 categories<jupyter_code>selected_categories = ['TOPS', 'SWEATERS', 'PANTS', 'JEANS', 'SHIRTS', 'DRESSES', 'SHORTS', 'SKIRTS']
subset_indexes = []
for i in range(len(file_h5['input_category'])):
    if file_h5['input_category'][i] in selected_categories:
        subset_indexes.append(i)
print('subset count = %d/%d' % (len(subset_indexes), len(file_h5['input_description'])))
subset_indexes[:30]<jupyter_output>subset count = 164352/260490<jupyter_text>How many items per categories<jupyter_code>categories_count = np.zeros(len(selected_categories))

for i in subset_indexes:
    cat = file_h5['input_category'][i]
    
    for j in range(len(selected_categories)):
        if cat == selected_categories[j]:
            categories_count[j] += 1
            break

categories_count<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1515850144092219, 'fraction_numerical': 0.02478386167146974, 'mean_word_length': 6.3872340425531915, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '175363', 'license': 'MIT', 'n_tokens_mistral': 658, 'n_tokens_neox': 642, 'n_words': 116, 'path': 'code/notebook fashiongen dataset sub-categories.ipynb', 'repo_name': 'menardai/FashionGenAttnGAN', 'size': '1735'}