<jupyter_start><jupyter_text>Synthetic spectra generator<jupyter_code>#First reset sys path so we use the correct envoirnment
import sys
sys.path=['', 'C:\\Users\\bryan\\Anaconda3\\envs\\tensorflow_GPU\\python37.zip', 'C:\\Users\\bryan\\Anaconda3\\envs\\tensorflow_GPU\\DLLs', 'C:\\Users\\bryan\\Anaconda3\\envs\\tensorflow_GPU\\lib', 'C:\\Users\\bryan\\Anaconda3\\envs\\tensorflow_GPU', 'C:\\Users\\bryan\\AppData\\Roaming\\Python\\Python37\\site-packages', 'C:\\Users\\bryan\\Anaconda3\\envs\\tensorflow_GPU\\lib\\site-packages']

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
n_points = 640
import tensorflow as tf
import keras.backend as K
from keras.models import Model, Sequential, load_model
from keras.layers import Dense, Conv1D, Flatten, BatchNormalization, Activation, Dropout
from keras import regularizers
from datetime import datetime
physical_devices = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)
tf.keras.backend.clear_session()
model = Sequential()

model.add(BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None,input_shape = (n_points, 1)))

model.add(Activation('relu'))

model.add(Conv1D(128, activation = 'relu', kernel_size = (32)))
model.add(Conv1D(64, activation = 'relu', kernel_size = (16)))
model.add(Conv1D(16, activation = 'relu', kernel_size = (8)))
model.add(Conv1D(16, activation = 'relu', kernel_size = (8)))
model.add(Conv1D(16, activation = 'relu', kernel_size = (8)))
model.add(Dense(32, activation = 'relu', kernel_regularizer=regularizers.l1_l2(l1 = 0, l2=0.1)))
model.add(Dense(16, activation = 'relu', kernel_regularizer=regularizers.l1_l2(l1 = 0, l2=0.1)))
model.add(Flatten())
model.add(Dropout(.25))
model.add(Dense(n_points, activation='relu'))


model.compile(loss='mse', optimizer='Adam', metrics=['mean_absolute_error','mse','accuracy'])
model.summary()<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
batch_normalization (BatchNo (None, 640, 1)            4         
_________________________________________________________________
activation (Activation)      (None, 640, 1)            0         
_________________________________________________________________
conv1d (Conv1D)              (None, 609, 128)          4224      
_________________________________________________________________
conv1d_1 (Conv1D)            (None, 594, 64)           131136    
_________________________________________________________________
conv1d_2 (Conv1D)            (None, 587, 16)           8208      
_________________________________________________________________
conv1d_3 (Conv1D)            (None, 580, 16)           2064      
________________________________________________________[...]<jupyter_text>Training<jupyter_code>from Generate_Data import *
for dataset_num in range(1,10):
    #X, y = generate_datasets_for_Paper_1(dataset_num,50000)
    temp, y = generate_datasets_(dataset_num,50000)
    X = np.empty((50000,640,1))
    X[:,:,0]=temp
    
    if dataset_num == 1:
        a=1
        b='a'
    elif dataset_num == 2:
        a=1
        b='b'
    elif dataset_num == 3:
        a=1
        b='c'
    elif dataset_num == 4:
        a=2
        b='a'
    elif dataset_num == 5:
        a=2
        b='b'
    elif dataset_num == 6:
        a=2
        b='c'
    elif dataset_num == 7:
        a=3
        b='a'
    elif dataset_num == 8:
        a=3
        b='b'
    else:
        a=3
        b='c'
    

    history = model.fit(X, y,epochs=10, verbose = 1, validation_split=0.25, batch_size=256) 
    my_path='./data/'+str(a)+b+'PAPER1'
    model.save(my_path)
    hist_df = pd.DataFrame(history.history) 
    with open(my_path+'.csv', mode='w') as f:
        hist_df.to_csv(f)
    
    
    my_path='./data/'+str(a)+b+'PAPER1'
    #history = load_model(my_path)
    # plt.plot(history.history['loss']) 
    # plt.plot(history.history['val_loss'])
    # plt.title('Model loss')
    # plt.ylabel('Loss')
    # plt.xlabel('Epoch')
    # plt.legend(['Train', 'Test'], loc='upper left')
    # plt.show()


    size1=100
    name1='./data/' +str(a)+b+'Raman_spectrums_valid.csv'
    name2= './data/'+str(a)+b+'CARS_spectrums_valid.csv'
    RAMAN = pd.read_csv(name1)
    BCARS = pd.read_csv(name2)
    RAMAN = RAMAN.values[:,1:]
    BCARS = BCARS.values[:,1:]
    
    #interpolate 1000 samples down to 640 samples
    from scipy import interpolate 
    min_wavenumber = 0.1
    max_wavenumber = 2000
    n_points = 1000
    step = (max_wavenumber-min_wavenumber)/(n_points)
    wavenumber_axis = np.arange(min_wavenumber, max_wavenumber, step)
    f_RAMAN = interpolate.interp1d(wavenumber_axis, RAMAN, kind='cubic')
    f_BCARS = interpolate.interp1d(wavenumber_axis, BCARS, kind='cubic')

    n_points = 640
    step = (max_wavenumber-min_wavenumber)/(n_points)
    new_axis = np.arange(min_wavenumber, max_wavenumber, step)
    RAMAN = f_RAMAN(new_axis)
    BCARS = f_BCARS(new_axis)
    #end interpolation
    
    
    RESULTS = RAMAN[:,:]
    print(np.shape(BCARS))
    X = np.empty((1, n_points,1))

    for m in range(size1):
        X[0,:,0]=BCARS[m,:]
        yhat = model.predict(X, verbose =0)
        RESULTS[m,:]=yhat.flatten()
        #print(RESULTS[m,:])

    #    print(np.shape(yhat))
    #     f, a = plt.subplots(2,1, sharex=True)
    #     a[0].plot(X.flatten(), label = 'cars')
    #     a[1].plot(RAMAN[m,:].T+.7, label = 'true',c= 'g' )
    #     a[1].plot(yhat.flatten()+1.4, label = 'pred.',c='r')
    #     plt.subplots_adjust(hspace=0)

    pd.DataFrame(RESULTS).to_csv('./data/' +str(a)+b+'Raman_spectrums_valid_PAPER1.csv')<jupyter_output>Epoch 1/10
  1/147 [..............................] - ETA: 0s - loss: 3.9206 - mean_absolute_error: 0.0218 - mse: 0.0045 - accuracy: 0.0039WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0120s vs `on_train_batch_end` time: 0.0199s). Check your callbacks.
147/147 [==============================] - 6s 39ms/step - loss: 2.1497 - mean_absolute_error: 0.0175 - mse: 0.0023 - accuracy: 0.2147 - val_loss: 0.9652 - val_mean_absolute_error: 0.0130 - val_mse: 0.0017 - val_accuracy: 0.3613
Epoch 2/10
147/147 [==============================] - 5s 37ms/step - loss: 0.4949 - mean_absolute_error: 0.0112 - mse: 9.7901e-04 - accuracy: 0.5041 - val_loss: 0.1960 - val_mean_absolute_error: 0.0101 - val_mse: 9.1942e-04 - val_accuracy: 0.5448
Epoch 3/10
147/147 [==============================] - 5s 37ms/step - loss: 0.0919 - mean_absolute_error: 0.0084 - mse: 5.4674e-04 - accuracy: 0.6209 - val_loss: 0.0309 - val_mean_absolute_error: 0.0079 - val_mse[...]<jupyter_text>Use this function to test the model on single instances<jupyter_code>def predict_and_plot():

    
    temp, y = generate_datasets_(dataset_num,1)
    X = np.empty((1,640,1))
    X[:,:,0]=temp
    yhat = model.predict(X, verbose =0)
    f, a = plt.subplots(2,1, sharex=True)
    a[0].plot(X.flatten(), label = 'cars')
    a[1].plot(y.T+.7, label = 'true',c= 'g' )
    a[1].plot(yhat.flatten()+1.4, label = 'pred.',c='r')
    plt.subplots_adjust(hspace=0)
    #return x, y.flatten(), yhat.flatten(), chi3, NRB
predict_and_plot()
# #predict_and_plot()
# for dataset_num in range(1,2):
#     if dataset_num == 1:
#         a=1
#         b='a'
#     elif dataset_num == 2:
#         a=1
#         b='b'
#     elif dataset_num == 3:
#         a=1
#         b='c'
#     elif dataset_num == 4:
#         a=2
#         b='a'
#     elif dataset_num == 5:
#         a=2
#         b='b'
#     elif dataset_num == 6:
#         a=2
#         b='c'
#     elif dataset_num == 7:
#         a=3
#         b='a'
#     elif dataset_num == 8:
#         a=3
#         b='b'
#     else:
#         a=3
#         b='c'
#     my_path='./data/'+str(a)+b+'PAPER1'
#     history = load_model(my_path)
#     # plt.plot(history.history['loss']) 
#     # plt.plot(history.history['val_loss'])
#     # plt.title('Model loss')
#     # plt.ylabel('Loss')
#     # plt.xlabel('Epoch')
#     # plt.legend(['Train', 'Test'], loc='upper left')
#     # plt.show()


#     size1=10
#     name1='./data/' +str(a)+b+'Raman_spectrums_valid.csv'
#     name2= './data/'+str(a)+b+'CARS_spectrums_valid.csv'
#     RAMAN = pd.read_csv(name1)
#     BCARS = pd.read_csv(name2)
#     RAMAN = RAMAN.values[:,1:]
#     BCARS = BCARS.values[:,1:]
    
    
#     from scipy import interpolate 
#     min_wavenumber = 0.1
#     max_wavenumber = 2000
#     n_points = 1000
#     step = (max_wavenumber-min_wavenumber)/(n_points)
#     wavenumber_axis = np.arange(min_wavenumber, max_wavenumber, step)
#     f_RAMAN = interpolate.interp1d(wavenumber_axis, RAMAN, kind='cubic')
#     f_BCARS = interpolate.interp1d(wavenumber_axis, BCARS, kind='cubic')

#     n_points = 640
#     step = (max_wavenumber-min_wavenumber)/(n_points)
#     new_axis = np.arange(min_wavenumber, max_wavenumber, step)
#     RAMAN = f_RAMAN(new_axis)
#     BCARS = f_BCARS(new_axis)
#     #end interpolation
    
    
    
#     RESULTS = RAMAN[:,:]
#     #print(np.shape(BCARS))
#     X = np.empty((1, n_points,1))

#     for m in range(size1):
#         X[0,:,0]=BCARS[m,:]
#         yhat = model.predict(X, verbose =0)
#         RESULTS[m,:]=yhat.flatten()
#         #print(RESULTS[m,:])

#     #    print(np.shape(yhat))
#         f, a = plt.subplots(2,1, sharex=True)
#         a[0].plot(X.flatten(), label = 'cars')
#         a[1].plot(RAMAN[m,:].T+.7, label = 'true',c= 'g' )
#         a[1].plot(yhat.flatten()+1.4, label = 'pred.',c='r')
#         plt.subplots_adjust(hspace=0)

#     #pd.DataFrame(RESULTS).to_csv('./data/' +str(a)+b+'Raman_spectrums_valid_PAPER1.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17338350331716013, 'fraction_numerical': 0.04861867511634815, 'mean_word_length': 3.2136003337505215, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '500141', 'license': 'MIT', 'n_tokens_mistral': 4069, 'n_tokens_neox': 3714, 'n_words': 638, 'path': 'Training_SPECNET/Code_to_train_Specnet.ipynb', 'repo_name': 'villawang/VECTOR-CARS', 'size': '10099'}