<jupyter_start><jupyter_text>Tengu Testing<jupyter_code>import numpy as np
from matplotlib import pyplot as plt
from scipy.stats import gaussian_kde, pearsonr
from functools import partial

from astropy.table import Table
from astropy.io import fits

import galsim
import ngmix

import os
import time
import tensorflow as tf
from tensorflow.keras.callbacks import TensorBoard
#from keras.backend.tensorflow_backend import set_session
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.compat.v1.Session(config=config)

#keras only after gpu ID and memory usage is set
import tensorflow.keras.models as km
import tensorflow.keras.layers as kl
import tensorflow.keras.regularizers as kr
import tensorflow.keras.backend as K
import tensorflow.keras.optimizers as ko

#from keras.utils.training_utils import multi_gpu_model
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Conv2D, AveragePooling2D, Input, Dense, Activation
from tensorflow.keras.layers import BatchNormalization, GlobalAveragePooling2D

print("Tensorflow version: {}".format(tf.version.VERSION))
import tensorflow_datasets as tfds
def Ribli19(imsize=50, n_target=2 ,n_channels=1, nf=64, reg = 5e-5,
          padding='same', gpu='0'):

    #input
    inp = kl.Input((imsize, imsize,n_channels))
    
    # conv block 1
    x = kl.Conv2D(nf, (3, 3), padding=padding,kernel_regularizer=kr.l2(reg))(inp)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.Conv2D(nf, (3, 3), padding=padding,kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.MaxPooling2D(strides=(2,2))(x)
    
    # conv block 2
    x = kl.Conv2D(2*nf, (3, 3), padding=padding,kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.Conv2D(2*nf, (3, 3), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.MaxPooling2D(strides=(2,2))(x)

    # conv block 3
    x = kl.Conv2D(4*nf, (3, 3), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.Conv2D(2*nf, (1, 1), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.Conv2D(4*nf, (3, 3), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    x = kl.MaxPooling2D(strides=(2,2))(x)

    # conv block 4
    x = kl.Conv2D(8*nf, (3, 3), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.Conv2D(4*nf, (1, 1), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.Conv2D(8*nf, (3, 3), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    x = kl.MaxPooling2D(strides=(2,2))(x)
    
    # conv block 5
    x = kl.Conv2D(16*nf, (3, 3), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.Conv2D(8*nf, (1, 1), padding=padding,  kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    x = kl.Conv2D(16*nf, (3, 3), padding=padding, kernel_regularizer=kr.l2(reg))(x)
    x = kl.Activation('relu')(kl.BatchNormalization()(x))
    
    #  end of conv
    
    x = kl.GlobalAveragePooling2D()(x)    
    x = kl.Dense(n_target, name = 'final_dense_n%d_ngpu%d' % (n_target, len(gpu.split(','))))(x)  

    model = km.Model(inputs=inp, outputs=x)  # make model
       
    return model
(datat, datav), ds_info = tfds.load('inverse_cat',as_supervised=True,split=['train[:70%]', 'train[70%:]'],with_info=True)
datat = datat.cache()
datat = datat.shuffle(ds_info.splits['train'].num_examples)
datat = datat.batch(128)
datat = datat.prefetch(tf.data.experimental.AUTOTUNE)
datav = datav.cache()
datav = datav.shuffle(ds_info.splits['train'].num_examples)
datav = datav.batch(128)
datav = datav.prefetch(tf.data.experimental.AUTOTUNE)
model = Ribli19(50,n_channels=1)
sgd = ko.SGD(lr=0.005, decay=0., momentum=0.9, nesterov=True)
model.compile(loss='mse',optimizer=sgd, metrics=['mae','mse'])

model.fit(datat,
          epochs=20,
          validation_data=datav,
          verbose=1)

sgd = ko.SGD(lr=0.0005, decay=0., momentum=0.9, nesterov=True)
model.compile(loss='mse',optimizer=sgd, metrics=['mae','mse'])

model.fit(datat,
          epochs=10,
          validation_data=datav,
          verbose=1)

sgd = ko.SGD(lr=0.00005, decay=0., momentum=0.9, nesterov=True)
model.compile(loss='mse',optimizer=sgd, metrics=['mae','mse'])

model.fit(datat,
          epochs=10,
          validation_data=datav,
          verbose=1)<jupyter_output>Epoch 1/20
547/547 [==============================] - 983s 2s/step - loss: 0.3053 - mae: 0.1849 - mse: 0.1119 - val_loss: 1.3948 - val_mae: 0.8454 - val_mse: 1.2021
Epoch 2/20
547/547 [==============================] - 966s 2s/step - loss: 0.1955 - mae: 0.0447 - mse: 0.0030 - val_loss: 0.1937 - val_mae: 0.0354 - val_mse: 0.0020
Epoch 3/20
153/547 [=======>......................] - ETA: 10:17 - loss: 0.1944 - mae: 0.0427 - mse: 0.0027
{'fraction_non_alphanumeric': 0.17746478873239438, 'fraction_numerical': 0.049014084507042255, 'mean_word_length': 4.2164544564152795, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '279525', 'license': 'MIT', 'n_tokens_mistral': 2239, 'n_tokens_neox': 2095, 'n_words': 367, 'path': 'notebooks/.ipynb_checkpoints/tengu_testing-checkpoint.ipynb', 'repo_name': 'andrevitorelli/TenGU', 'size': '5325'}