<jupyter_start><jupyter_text>Synthetic spectra generator<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
max_features = 15
n_points = 640
nu = np.linspace(0,1,n_points)
def random_chi3():
    """
    generates a random spectrum, without NRB. 
    output:
        params =  matrix of parameters. each row corresponds to the [amplitude, resonance, linewidth] of each generated feature (n_lor,3)
    """
    n_lor = np.random.randint(1,max_features)
    a = np.random.uniform(0,1,n_lor)
    w = np.random.uniform(0,1,n_lor)
    g = np.random.uniform(0.001,0.008, n_lor)
    
    params = np.c_[a,w,g]
    return params
def build_chi3(params):
    """
    buiilds the normalized chi3 complex vector
    inputs: 
        params: (n_lor, 3)
    outputs
        chi3: complex, (n_points, )
    """
    
    chi3 = np.sum(params[:,0]/(-nu[:,np.newaxis]+params[:,1]-1j*params[:,2]),axis = 1)
    
    return chi3/np.max(np.abs(chi3))  
def sigmoid(x,c,b):
	return 1/(1+np.exp(-(x-c)*b))
def generate_nrb():
    """
    Produces a normalized shape for the NRB
    outputs
        NRB: (n_points,)
    """
    bs = np.random.normal(10,5,2)
    c1 = np.random.normal(0.2,0.3)
    c2 = np.random.normal(0.7,.3)
    cs = np.r_[c1,c2]
    sig1 = sigmoid(nu, cs[0], bs[0])
    sig2 = sigmoid(nu, cs[1], -bs[1])
    nrb  = sig1*sig2
    return nrb
def get_spectrum():
    """
    Produces a cars spectrum.
    It outputs the normalized cars and the corresponding imaginary part.
    Outputs
        cars: (n_points,)
        chi3.imag: (n_points,)
    """
    chi3 = build_chi3(random_chi3())*np.random.uniform(0.3,1)
    nrb = generate_nrb()
    noise = np.random.randn(n_points)*np.random.uniform(0.0005,0.003)
    cars = ((np.abs(chi3+nrb)**2)/2+noise)
    return cars, chi3.imag
import tensorflow as tf
import keras.backend as K
from keras.models import Model, Sequential
from keras.layers import Dense, Conv1D, Flatten, BatchNormalization, Activation, Dropout
from keras import regularizers
from datetime import datetime
physical_devices = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)
tf.keras.backend.clear_session()
model = Sequential()

model.add(BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None,input_shape = (n_points, 1)))

model.add(Activation('relu'))

model.add(Conv1D(128, activation = 'relu', kernel_size = (32)))
model.add(Conv1D(64, activation = 'relu', kernel_size = (16)))
model.add(Conv1D(16, activation = 'relu', kernel_size = (8)))
model.add(Conv1D(16, activation = 'relu', kernel_size = (8)))
model.add(Conv1D(16, activation = 'relu', kernel_size = (8)))
model.add(Dense(32, activation = 'relu', kernel_regularizer=regularizers.l1_l2(l1 = 0, l2=0.1)))
model.add(Dense(16, activation = 'relu', kernel_regularizer=regularizers.l1_l2(l1 = 0, l2=0.1)))
model.add(Flatten())
model.add(Dropout(.25))
model.add(Dense(n_points, activation='relu'))


model.compile(loss='mse', optimizer='Adam', metrics=['mean_absolute_error','mse','accuracy'])
model.summary()<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>def generate_batch(size = 10000):
    X = np.empty((size, n_points,1))
    y = np.empty((size,n_points))
    
    for i in range(size):
        X[i,:,0], y[i,:] = get_spectrum()
    return X, y
X, y = generate_batch(50000)
history = model.fit(X, y,epochs=10, verbose = 1, validation_split=0.25, batch_size=256) 
plt.plot(history.history['loss']) 
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()<jupyter_output><empty_output><jupyter_text>Use this function to test the model on single instances<jupyter_code>def predict_and_plot():
    x,y = generate_batch(1)
    yhat = model.predict(x, verbose =0)
    f, a = plt.subplots(2,1, sharex=True)
    a[0].plot(x.flatten(), label = 'cars')
    a[1].plot(y.T+.7, label = 'true',c= 'g' )
    a[1].plot(yhat.flatten()+1.4, label = 'pred.',c='r')
    plt.subplots_adjust(hspace=0)
    #return x, y.flatten(), yhat.flatten(), chi3, NRB<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16056402092335684, 'fraction_numerical': 0.03775301341823971, 'mean_word_length': 4.553030303030303, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '11718', 'license': 'MIT', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1649, 'n_words': 321, 'path': 'Specnet_Published.ipynb', 'repo_name': 'Valensicv/Specnet', 'size': '4397'}