<jupyter_start><jupyter_text>[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/shmouses/EELSpecNet/blob/main/article/EELSpecNet_GPU_10D.ipynb)   **EELSpecNet**Dependencies in the requirements.txt fileMain code generating EELSpecNet Neural Networks with different depths and training them.<jupyter_code>import tensorflow as tf
import os 
import tensorflow_datasets as tfds
import copy
import numpy as np
from scipy import signal
from scipy.special import wofz, erf
import matplotlib.pyplot as plt
import notebook
from tqdm.auto import tqdm
import random
import os
import pandas as pd

import tensorflow.experimental.numpy as tnp
tnp.experimental_enable_numpy_behavior()

import notebook
from bokeh.plotting import figure, show
from bokeh.io import output_notebook 
output_notebook()
# Specific to the use of GPUs in Google Colab

%tensorflow_version 2.x

device_name = tf.test.gpu_device_name()
if device_name != '/device:GPU:0':
  raise SystemError('GPU device not found')
print('Found GPU at: {}'.format(device_name))<jupyter_output><empty_output><jupyter_text>**Neural Networks design**<jupyter_code>class EELSpecNetModel_CNN_10D(tf.keras.Model):
    
    
    def __init__(self, ene_dim):
        super(EELSpecNetModel_CNN_10D, self).__init__()
        
        kerl_size = 4
 
        self.conv_1024x64 = tf.keras.layers.Conv2D(64, (1,kerl_size), strides = (1,2),
                                                   activation = 'relu', padding = 'same',
                                                   kernel_initializer='random_uniform')
 
        self.conv_512x128 = tf.keras.layers.Conv2D(128, (1,kerl_size), strides = (1,2),
                                                   activation = 'relu', padding = 'same',
                                                   kernel_initializer='random_uniform')
 
        self.conv_256x256 = tf.keras.layers.Conv2D(256, (1,kerl_size), strides = (1,2),
                                                   activation = 'relu',padding = 'same',
                                                   kernel_initializer='random_uniform')
 
        self.conv_128x512 = tf.keras.layers.Conv2D(512, (1,kerl_size), strides = (1,2),
                                                   activation = 'relu', padding = 'same',
                                                   kernel_initializer='random_uniform')
 
        self.conv_64x1024 = tf.keras.layers.Conv2D(1024, (1,kerl_size), strides = (1,2),
                                                   activation = 'relu', padding = 'same',
                                                   kernel_initializer='random_uniform')
 
        self.conv_32x2048 = tf.keras.layers.Conv2D(2048, (1,kerl_size), strides = (1,2),
                                                   activation = 'relu',padding = 'same',
                                                   kernel_initializer='random_uniform')
 
        self.conv_16x2048 = tf.keras.layers.Conv2D(2048, (1,kerl_size), strides = (1,2),
                                                   activation = 'relu', padding = 'same',
                                                   kernel_initializer='random_uniform')
 
        self.conv_8x2048 = tf.keras.layers.Conv2D(2048, (1,kerl_size), strides = (1,2),
                                                  activation = 'relu', padding = 'same',
                                                  kernel_initializer='random_uniform')
 
        self.conv_4x2048 = tf.keras.layers.Conv2D(2048, (1,kerl_size), strides = (1,2),
                                                  activation = 'relu',padding = 'same',
                                                  kernel_initializer='random_uniform')
 
        self.conv_2x2048 = tf.keras.layers.Conv2D(2048, (1,kerl_size), strides = (1,2),
                                                  activation = 'relu', padding = 'same',
                                                  kernel_initializer='random_uniform')
        
        #=======================================================================
 
        self.deconv_4x2048 = tf.keras.layers.Conv2DTranspose(2048, (1,kerl_size), strides = (1,2),
                                                             activation = 'relu', padding = 'same',
                                                             kernel_initializer='random_uniform')
 
        self.deconv_8x2048 = tf.keras.layers.Conv2DTranspose(2048, (1,kerl_size), strides = (1,2),
                                                             activation = 'relu', padding = 'same',
                                                             kernel_initializer='random_uniform')
 
        self.deconv_16x2048 = tf.keras.layers.Conv2DTranspose(2048, (1,kerl_size), strides = (1,2),
                                                              activation = 'relu', padding = 'same',
                                                              kernel_initializer='random_uniform')
 
        self.deconv_32x2048 = tf.keras.layers.Conv2DTranspose(2048, (1,kerl_size), strides = (1,2),
                                                              activation = 'relu', padding = 'same',
                                                              kernel_initializer='random_uniform')
 
        self.deconv_64x1024 = tf.keras.layers.Conv2DTranspose(1024, (1,kerl_size), strides = (1,2),
                                                              activation = 'relu', padding = 'same',
                                                              kernel_initializer='random_uniform')
 
        self.deconv_128x512 = tf.keras.layers.Conv2DTranspose(512, (1,kerl_size), strides = (1,2),
                                                              activation = 'relu', padding = 'same',
                                                              kernel_initializer='random_uniform')
 
        self.deconv_256x256 = tf.keras.layers.Conv2DTranspose(256, (1,kerl_size), strides = (1,2),
                                                              activation = 'relu', padding = 'same',
                                                              kernel_initializer='random_uniform')
        
        self.deconv_512x128 = tf.keras.layers.Conv2DTranspose(128, (1,kerl_size), strides = (1,2),
                                                              activation = 'relu', padding = 'same',
                                                              kernel_initializer='random_uniform')
        
        self.deconv_1024x64 = tf.keras.layers.Conv2DTranspose(64, (1,kerl_size), strides = (1,2),
                                                              activation = 'relu', padding = 'same',
                                                              kernel_initializer='random_uniform')
        
        self.deconv_2048x1 = tf.keras.layers.Conv2DTranspose(1, (1,kerl_size), strides = (1,2),
                                                             activation = 'tanh', padding = 'same',
                                                             kernel_initializer='random_uniform')
        
        self.concat = tf.keras.layers.concatenate
        self.relu = tf.keras.activations.relu
        
    
    
    def call(self, inputs):
        
        enc_1024x64 = self.conv_1024x64(inputs)
        
        enc_512x128 = self.conv_512x128(enc_1024x64)
        
        enc_256x256 = self.conv_256x256(enc_512x128)
        
        enc_128x512 = self.conv_128x512(enc_256x256)
        
        enc_64x1024 = self.conv_64x1024(enc_128x512)
        
        enc_32x2048 = self.conv_32x2048(enc_64x1024)
        
        enc_16x2048 = self.conv_16x2048(enc_32x2048)
        
        enc_8x2048 = self.conv_8x2048(enc_16x2048)
        
        enc_4x2048 = self.conv_4x2048(enc_8x2048)
                
        enc_2x2048 = self.conv_2x2048(enc_4x2048)
        
        #=======================================================================
 
        dcd_4x2048 = self.deconv_4x2048(enc_2x2048)  
        dcd_4x2048x2 = self.concat([dcd_4x2048, enc_4x2048], axis=-1)
        
        dcd_8x2048 = self.deconv_8x2048(dcd_4x2048x2)
        dcd_8x2048x2 = self.concat([dcd_8x2048, enc_8x2048], axis=-1)
                
        dcd_16x2048 = self.deconv_16x2048(dcd_8x2048x2)
        dcd_16x2048x2 = self.concat([dcd_16x2048, enc_16x2048], axis=-1)
        
        dcd_32x2048 = self.deconv_32x2048(dcd_16x2048x2)
        dcd_32x2048x2 = self.concat([dcd_32x2048, enc_32x2048], axis=-1)
        
        dcd_64x1024 = self.deconv_64x1024(dcd_32x2048x2)
        dcd_64x1024x2 = self.concat([dcd_64x1024, enc_64x1024], axis=-1)
        
        dcd_128x512 = self.deconv_128x512(dcd_64x1024x2)
        dcd_128x512x2 = self.concat([dcd_128x512, enc_128x512], axis=-1)
        
        dcd_256x256 = self.deconv_256x256(dcd_128x512x2)
        dcd_256x256x2 = self.concat([dcd_256x256, enc_256x256], axis=-1)
        
        dcd_512x128 = self.deconv_512x128(dcd_256x256x2)
        dcd_512x128x2 = self.concat([dcd_512x128, enc_512x128], axis=-1)
        
        dcd_1024x64 = self.deconv_1024x64(dcd_512x128x2)
        dcd_1024x64x2 = self.concat([dcd_1024x64, enc_1024x64], axis=-1)
        
        dcd_2048x1 = self.deconv_2048x1(dcd_1024x64x2)
 
        
        return(dcd_2048x1)<jupyter_output><empty_output><jupyter_text>**Load data for model training**  In this notebook, it is assumed that the training data ("convolved_general_6000.npy" and "original_general_6000.npy" files) are already available in the training folder. If needed the training data can be generated in the training folder executing the Generate_Training_Set.ipynb notebook. Once generated make sure the data is saved in the training folder to be properly loaded below.<jupyter_code># initial data
data_convolved_loaded = np.load("training/convolved_general_6000.npy")
tnp_convolved_loaded = tnp.asarray(data_convolved_loaded)

# target data
data_original_loaded = np.load("training/original_general_6000.npy")
tnp_original_loaded = tnp.asarray(data_original_loaded)<jupyter_output><empty_output><jupyter_text>**Prepare training data**<jupyter_code>x_dim, e_dim = np.shape(data_original_loaded)

# Avoid zeros in data
tnp_original_loaded += 0.001      
tnp_convolved_loaded += 0.001   

tnp_data_original = tnp_original_loaded.reshape((x_dim, 1, e_dim, 1))
tnp_data_convolved = tnp_convolved_loaded.reshape((x_dim, 1, e_dim, 1))
tnp_train_original = tnp_data_original[:, :, :, :]
tnp_train_convolved = tnp_data_convolved[:, :, :, :]<jupyter_output><empty_output><jupyter_text>**Instantiate, build and train model**<jupyter_code>model = EELSpecNetModel_CNN_10D(2048)
op = tf.keras.optimizers.Adam(learning_rate = 5e-5)
model.compile(optimizer = op, loss = 'BinaryCrossentropy', metrics = ['mape','mse'])
model.build((1,1,2048,1))
model.summary()
# Training
history= model.fit(tnp_train_convolved, tnp_train_original, validation_split=0.16, batch_size= 16, epochs = 1000)
print("------------------------ Training done !!! ------------------------")<jupyter_output><empty_output><jupyter_text>**Save model**<jupyter_code>model.save('model/general_6000') # Warning model size 2.44Gb - not stored in github repository

# Saving training history to csv:
hist_df = pd.DataFrame(history.history) 
hist_csv_file = 'model/history_general_6000.csv'
with open(hist_csv_file, mode='w') as f:
    hist_df.to_csv(f)

# Saving training history to numpy file:
np.save('model/Numpy_history_general_6000.npy', history.history)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11227402473834443, 'fraction_numerical': 0.07456102413286048, 'mean_word_length': 1.8761194029850747, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 80, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '161408', 'license': 'MIT', 'n_tokens_mistral': 3916, 'n_tokens_neox': 3380, 'n_words': 553, 'path': 'notebook/EELSpecNet_GPU_10D.ipynb', 'repo_name': 'shmouses/EELSpecNet', 'size': '11561'}