<jupyter_start><jupyter_text>Code to save a few 3D images from the datasetOct 15, 2020<jupyter_code>import numpy as np
import h5py
import os

import glob
import time

from scipy.ndimage import gaussian_filter   ### For gausian filter
### Location of hdf5 files
data_dir='/global/project/projectdirs/m3363/www/cosmoUniverse_2019_08_const/'
### Extract list of hdf5 files
f_list=glob.glob(data_dir+'*.hdf5')
len(f_list)
slices = []

for fname in f_list[:10]:
    with h5py.File(fname, 'r') as inputdata:
        img_arr=np.array(inputdata['full'])
        slices.append(img_arr[:,:,:,0])
        
print(len(slices))
slices=np.array(slices)
print(slices.shape)
### Save 3D images to file
np.save('3d_images_10.npy',slices)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12817089452603472, 'fraction_numerical': 0.03871829105473965, 'mean_word_length': 5.25, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '374228', 'license': 'BSD-3-Clause-LBNL', 'n_tokens_mistral': 310, 'n_tokens_neox': 294, 'n_words': 63, 'path': 'code/0a_data_preprocessing/3_get_3d_images.ipynb', 'repo_name': 'vmos1/cosmogan_pytorch', 'size': '749'}