<jupyter_start><jupyter_text>Alle auf einmal in Liste und Array<jupyter_code>train_image = []
for i in tqdm(range(train.shape[0])):
    img = image.load_img(img_dir+train['filename'][i])
    img = image.img_to_array(img)
    img = img/255
    train_image.append(img)
X = np.array(train_image)<jupyter_output><empty_output><jupyter_text>Save memory with multiple lists<jupyter_code># test for memory saving conversion online
first = []
for i in range(0,2000):
    img = image.load_img(img_dir+train['filename'][i])
    img = image.img_to_array(img)
    img = img/255
    first.append(img)
second = []
for i in range(2000,3000):
    img = image.load_img(img_dir+train['filename'][i])
    img = image.img_to_array(img)
    img = img/255
    second.append(img)
third = []
for i in range(3000,4704):
    img = image.load_img(img_dir+train['filename'][i])
    img = image.img_to_array(img)
    img = img/255
    third.append(img)
liste = first+second+third
len(liste)
# Check
im = liste[-1]
from matplotlib import pyplot as plt
plt.imshow(im, interpolation='nearest')
plt.show()
X = np.array(liste)<jupyter_output><empty_output><jupyter_text>Save memory with hdf5<jupyter_code>import sys
import glob
import h5py
import cv2
IMG_WIDTH = 512
IMG_HEIGHT = 512

h5file = 'import_images.h5'

nfiles = len(glob.glob(img_dir))
print(f'count of image files nfiles={nfiles}')

# resize all images and load into a single dataset
with h5py.File(h5file,'w') as  h5f:
    img_ds = h5f.create_dataset('images',shape=(nfiles, IMG_WIDTH, IMG_HEIGHT,3), dtype=int)
    for cnt, ifile in enumerate(glob.iglob('./*.ppm')) :
        img = cv2.imread(ifile, cv2.IMREAD_COLOR)
        # or use cv2.IMREAD_GRAYSCALE, cv2.IMREAD_UNCHANGED
        img_resize = cv2.resize( img, (IMG_WIDTH, IMG_HEIGHT) )
        img_ds[cnt:cnt+1:,:,:] = img_resize
IMG_WIDTH = 512
IMG_HEIGHT = 512
h5file = 'import_images.h5'
nfiles = len(glob.glob('D:/data/Projects/notebooks/RetinaAI/train_resampled/*.jpg'))
print(f'Count of image files nfiles = {nfiles}')
with h5py.File(h5file,'w') as  h5f:
    img_ds = h5f.create_dataset('images',shape=(nfiles, IMG_WIDTH, IMG_HEIGHT,3), dtype=int)
    for cnt, ifile in enumerate(glob.iglob('D:/data/Projects/notebooks/RetinaAI/train_resampled/*.jpg')) :
        img = cv2.imread(ifile, cv2.IMREAD_COLOR)
        # or use cv2.IMREAD_GRAYSCALE, cv2.IMREAD_UNCHANGED
        img_resize = cv2.resize( img, (IMG_WIDTH, IMG_HEIGHT) )
        img_ds[cnt:cnt+1:,:,:] = img_resize
with h5py.File(h5file,'w') as h5f:
    img_ds = h5f.create_dataset('images',shape=(nfiles, IMG_WIDTH, IMG_HEIGHT,3), dtype=int)
    for i in tqdm(range(train.shape[0])):
    img = image.load_img(img_dir+train['filename'][i])
    img = image.img_to_array(img)
    img = img/255<jupyter_output><empty_output><jupyter_text>Save and Load npz file<jupyter_code>from numpy import savez_compressed
savez_compressed('data.npz', data)
from numpy import load
# load dict of arrays
dict_data = load('data.npz')
# extract the first array
data = dict_data['arr_0']
# print the array
print(data)<jupyter_output>[[[[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  ...

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]]


 [[[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  ...

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]]

  [[0. 0. 0.]
   [0. 0. 0.]
   [0. 0. 0.]
   ...
   [0[...]<jupyter_text>Check Images<jupyter_code># No of images, height, width, depth
X.shape
# Pixel intensity
X[300].min(), X[300].max(), X[300].mean()
X[300].shape, X[300].size
X[300][100][200]
plt.imshow(X[300])
y = np.array(train.drop(['filename', 'new'], axis=1))
y.shape
y
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=2, test_size=0.1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1988330341113106, 'fraction_numerical': 0.06889587073608618, 'mean_word_length': 2.6147607461476077, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '133901', 'license': 'MIT', 'n_tokens_mistral': 2194, 'n_tokens_neox': 1990, 'n_words': 453, 'path': '01_Exploration and Preprocessing/Images2Array.ipynb', 'repo_name': 'daimonae/retina', 'size': '4456'}