<jupyter_start><jupyter_text>Print DependenciesDependences are fundamental to record the computational environment.<jupyter_code>%load_ext watermark

# python, ipython, packages, and machine characteristics
%watermark -v -m -p pandas,keras,numpy,math,tensorflow,matplotlib,h5py,progressbar,scipy

# date
print (" ")
%watermark -u -n -t -z<jupyter_output>Python implementation: CPython
Python version       : 3.7.7
IPython version      : 7.19.0

pandas     : 1.2.0
keras      : 2.4.3
numpy      : 1.19.5
math       : unknown
tensorflow : 2.4.0
matplotlib : 3.3.3
h5py       : 2.10.0
progressbar: 2.5
scipy      : 1.6.0

Compiler    : GCC 5.4.0 20160609
OS          : Linux
Release     : 5.8.0-41-generic
Machine     : x86_64
Processor   : x86_64
CPU cores   : 8
Architecture: 64bit

 
Last updated: Tue Feb 02 2021 18:21:47CET<jupyter_text>Load of the data<jupyter_code>from process import loaddata
regr_data = loaddata("../data/regression/250.csv")
class_data = loaddata("../data/classifier/250.csv")
np.random.shuffle(regr_data)
y = regr_data[:,-3:]
x = regr_data[:,:6]
np.random.shuffle(class_data)
yc = class_data[:,0]
xc = class_data[:,1:]
train_split = 0.75
train_limit = int(len(y)*train_split)
print("Training sample: {0} \nValuation sample: {1}".format(train_limit, len(y)-train_limit))
x_train = x[:train_limit]
x_val = x[train_limit:]

y_train = y[:train_limit]
y_val = y[train_limit:]<jupyter_output><empty_output><jupyter_text>Model Build<jupyter_code>from keras.models import Sequential
from keras.layers.core import Dense
import keras.backend as K
from keras import optimizers
from keras import models
from keras import layers
from keras import regularizers<jupyter_output><empty_output><jupyter_text>!! The dropout 0.2 seems to work better<jupyter_code>def build_model() :
    model = models.Sequential()
    model.add (layers.Dense (12, kernel_initializer= "normal" , input_shape = x.shape))
    model.add (layers.Dense (32, activation = "relu"))
    model.add (layers.Dense (64, activation = "relu"))
    model.add (layers.Dense (128, activation = "relu"))
    model.add (layers.Dense (128, activation = "relu"))
    model.add (layers.Dense (64, activation = "relu"))
    model.add (layers.Dense (32, activation = "relu"))
    model.add (layers.Dense (12, activation = "relu"))
    model.add (layers.Dense (3))
    model.compile(optimizer = "rmsprop" , loss = "mse" , metrics =["mape"])
    return model
model = build_model ()
history = model.fit ( x_train, y_train, epochs = 20, batch_size = 50, validation_data = (x_val, y_val) )
model.save("../models/regression/large_mse250.h5")
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(loss) + 1)

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>Test spectrum<jupyter_code>from tensorflow import keras
model_regr = keras.models.load_model('../models/regression/large_mse250.h5')
model_class = keras.models.load_model('../models/classifier/with-dropout-250.h5')
model_regr.summary()
model_class.summary()
def energy_spectrum(energy_array, bins):
    energy_array = np.array(energy_array)
    plt.hist(energy_array, bins, histtype=u'step', density=True)
    plt.yscale("log")
    plt.show()
final_p = []
final_e = []

for regr_pair in regr_data:
    final_e.append(np.linalg.norm(regr_pair[10:13]))
    final_p.append(np.linalg.norm(regr_pair[13:16]))
y = []
pred = []

y = model_class.predict(xc)
data = np.hstack((y, xc))
data = data[np.logical_not(data[:,0] > 0.5)]
pred = model_regr.predict(data[:,1:7])
final_p_nn = []
for pred_ in pred:
    final_p_nn.append(np.linalg.norm(pred_))
energy_spectrum(photon_final_nn, 75)
from scipy.stats import norm
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import scipy.stats as stats
from scipy.stats import chisquare
mean,std=norm.fit(final_p)
plt.hist(final_p, bins=100, alpha = 0.5, color = 'mediumslateblue', label='Photon Spectrum', density = True)
plt.legend(loc='upper right')
xmin, xmax = plt.xlim()
x_p = np.linspace(xmin, xmax, 100)
y_p = norm.pdf(x_p, mean, std)
plt.plot(x_p, y_p,'g--', linewidth=2)
plt.show()
print('mean = ', mean)
print('std = ', std)
print("chi square = ", stats.chisquare(final_p))
mean_nn,std_nn=norm.fit(final_p_nn)
plt.hist(final_p_nn, bins=100, alpha = 0.5,  color = 'indianred', label='NN prediction', density = True)
xmin, xmax = plt.xlim()
x_nn = np.linspace(xmin, xmax, 100)
y_nn = norm.pdf(x_nn, mean_nn, std_nn)
plt.plot(x_nn, y_nn,'r--', linewidth=2)
plt.legend(loc='upper right')
plt.show()
print('mean = ', mean)
print('std = ', std)
print("chi square = ", stats.chisquare(final_p_nn))
plt.hist(final_p_nn, 100, alpha=0.5, color = 'indianred', label='NN prediction', density = True)
plt.hist(final_p, 100, alpha=0.5, color = 'mediumslateblue', label='Photon Spectrum', density = True)
plt.xlabel('Photon Momentum')
plt.ylabel('count')
plt.legend(loc='upper right')
x_nn = np.linspace(xmin, xmax, 100)
y_nn = norm.pdf(x_nn, mean_nn, std_nn)
plt.plot(x_nn, y_nn,'g--', label = 'fit NN', linewidth = 2)
plt.legend(loc='upper right')
x_p = np.linspace(xmin, xmax, 100)
y_p = norm.pdf(x_p, mean, std)
plt.plot(x_p, y_p, 'r:', label = 'fit Photon Spectrum', linewidth = 2)
plt.legend(loc = 'upper right')
plt.show()
fig, axs = plt.subplots(2, sharex = True)
fig.suptitle('Vertically stacked subplots')
axs[0].hist(final_p_nn, 100, alpha=0.5, color = 'indianred', label='NN prediction', density = True)
axs[1].hist(final_p, 100, alpha=0.5, color = 'mediumslateblue', label='Photon Spectrum', density = True)
fig.show()<jupyter_output>/home/linuxbrew/.linuxbrew/opt/python/lib/python3.7/site-packages/ipykernel_launcher.py:5: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.
  """
{'fraction_non_alphanumeric': 0.1541931172736385, 'fraction_numerical': 0.032408954226528564, 'mean_word_length': 5.121676891615542, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '43723', 'license': 'MIT', 'n_tokens_mistral': 2450, 'n_tokens_neox': 2294, 'n_words': 492, 'path': 'notebooks/.ipynb_checkpoints/Comp_Scatt_NeuralNetwork_Regression-checkpoint.ipynb', 'repo_name': 'chiarabadiali/comp_scatt_ML', 'size': '5986'}