<jupyter_start><jupyter_text>test generator<jupyter_code>import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"   # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = ""
import keras
datagen = keras.preprocessing.image.ImageDataGenerator(
    rescale = 0
)
train_generator = datagen.flow_from_directory(
    collection.dir_valid+'/..',
    target_size=(45, 80),
    color_mode='rgb',
    classes=None,
    class_mode='input',
    batch_size=24, 
    shuffle=True,
    seed=None,
    save_to_dir=None,
    save_prefix='',
    save_format='jpg',
    follow_links=False,
    subset=None,
    interpolation='nearest'
)
np_arr_o = train_generator.next()[1]
np_arr_o.shape
np_arr_o
#np_arr = np.reshape(np_arr_o, (720, 1280, 3))
#np_arr[0][0]
np_arr = np_arr_o.astype('uint8')
PIL.Image.fromarray(np_arr[0], 'RGB')
img = Image.open('/home/jovyan/work/data/processed/models/autoencoder/valid/k/20170416-013001.jpg')
img.load()
data = np.asarray(img, dtype=np.uint8)
data.shape
data[0]
PIL.Image.fromarray(data, 'RGB')<jupyter_output><empty_output><jupyter_text>Test autoencoder<jupyter_code># https://blog.keras.io/building-autoencoders-in-keras.html
from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D, Reshape, Flatten
from keras.models import Model
from keras import backend as K

input_img = Input((45, 80, 3))  # adapt this if using `channels_first` image data format

x = Conv2D(1, (3, 3), activation='relu', padding='valid', name = "enc_conv_1", strides = 2)(input_img)
encoded = MaxPooling2D((2, 2), padding='same')(x)

# at this point the representation is (11, 20, 8) i.e. 480-dimensional

x = Conv2D(1, (3, 3), activation='relu', padding = 'same', name = "dec_conv_1")(encoded)
x = UpSampling2D((2, 2))(x)
x = Flatten()(x)
x = Dense(10800, input_shape = (48, 80, 1))(x)
decoded = Reshape((45, 80, 3))(x)
#  decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same', name = "dec_conv_4")(x)

autoencoder = Model(input_img, decoded)
autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy')
autoencoder.summary()
autoencoder.fit_generator(train_generator, steps_per_epoch=1000, epochs = 2)<jupyter_output>Epoch 1/2
 174/1000 [====>.........................] - ETA: 4:53 - loss: -1657.1020
{'fraction_non_alphanumeric': 0.16965085049239034, 'fraction_numerical': 0.052372426141450316, 'mean_word_length': 5.313559322033898, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '240102', 'license': 'MIT', 'n_tokens_mistral': 940, 'n_tokens_neox': 865, 'n_words': 163, 'path': 'notebooks/test_collection.ipynb', 'repo_name': 'prise6/smart-iss-posts', 'size': '2234'}