<jupyter_start><jupyter_text>Print DependenciesDependences are fundamental to record the computational environment.<jupyter_code>%load_ext watermark

# python, ipython, packages, and machine characteristics
%watermark -v -m -p pandas,keras,numpy,math,tensorflow,matplotlib,h5py,progressbar,scipy

# date
print (" ")
%watermark -u -n -t -z<jupyter_output>Python implementation: CPython
Python version       : 3.7.7
IPython version      : 7.19.0

pandas     : 1.2.0
keras      : 2.4.3
numpy      : 1.19.5
math       : unknown
tensorflow : 2.4.0
matplotlib : 3.3.3
h5py       : 2.10.0
progressbar: 2.5
scipy      : 1.6.0

Compiler    : GCC 5.4.0 20160609
OS          : Linux
Release     : 5.8.0-40-generic
Machine     : x86_64
Processor   : x86_64
CPU cores   : 8
Architecture: 64bit

 
Last updated: Fri Jan 29 2021 10:40:25CET<jupyter_text>Load of the data<jupyter_code>from process import loaddata
class_data = loaddata("../data/classifier/250.csv")
np.random.shuffle(class_data)
y = class_data[:,-7:-4]
x = class_data[:,1:7]
class_data[0,1:7]
class_data[0]
import numpy as np

arr = np.array([[1, 2, 3, 4, 5, 8, 14], [6, 7, 8, 9, 10, 8, 8]])

prova = arr[:,-7:-4]
print(prova)
train_split = 0.75
train_limit = int(len(y)*train_split)
print("Training sample: {0} \nValuation sample: {1}".format(train_limit, len(y)-train_limit))
x_train = x[:train_limit]
x_val = x[train_limit:]

y_train = y[:train_limit]
y_val = y[train_limit:]<jupyter_output><empty_output><jupyter_text>Model Build<jupyter_code>from keras.models import Sequential
from keras.layers.core import Dense
import keras.backend as K
from keras import optimizers
from keras import models
from keras import layers
from keras import regularizers<jupyter_output><empty_output><jupyter_text>!! The dropout 0.2 seems to work better<jupyter_code>def build_model() :
    model = models.Sequential()
    model.add (layers.Dense (6, kernel_initializer= "normal" , input_shape = x.shape))
    model.add (layers.Dense (12, activation = "relu"))
    model.add (layers.Dense (32, activation = "relu"))
    model.add (layers.Dense (64, activation = "relu"))
    model.add (layers.Dense (128, activation = "relu"))
    model.add (layers.Dense (128, activation = "relu"))
    model.add (layers.Dense (64, activation = "relu"))
    model.add (layers.Dense (32, activation = "relu"))
    model.add (layers.Dense (12, activation = "relu"))
    model.add (layers.Dense (6, activation = "relu"))
    model.add (layers.Dense (3))
    model.compile(optimizer = "rmsprop" , loss = "mse" , metrics =["mae"])
    return model
model = build_model ()
history = model.fit ( x_train, y_train, epochs = 20, batch_size = 150, validation_data = (x_val, y_val) )
model.save("../models/classificationandregression/large_mse250.h5")
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(loss) + 1)

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>Test spectrum<jupyter_code>from tensorflow import keras
model = keras.models.load_model('../models/classificationandregression/large_mse250.h5')
model.summary()
def energy_spectrum(energy_array, bins):
    energy_array = np.array(energy_array)
    plt.hist(energy_array, bins, histtype=u'step', density=True)
    plt.yscale("log")
    plt.show()
y
from tensorflow import keras 
photon_final_nn = []
prediction = model.predict(x)
p1e_nn = prediction[:,0] 
p1e = y[:,0]
print(p1e_nn)
print(p1e)
plt.hist(p1e_nn, 60, alpha=0.5, label='NN prediction', density = True)
plt.hist(p1e, 100, alpha=0.5, label='Photon Momentum from simulations', density = True)
plt.xlabel('Electron momentum x-direction')
plt.ylabel('count')
plt.legend(loc='upper right')
#plt.xlim((0, 0.25))
plt.ylim((0, 12))
plt.show()
p2e_nn = prediction[:,1] 
p2e = y[:,1]
print(p2e_nn)
print(p2e)
plt.hist(p2e_nn, 60, alpha=0.5, label='NN prediction', density = True)
plt.hist(p2e, 100, alpha=0.5, label='Photon Momentum from simulations', density = True)
plt.xlabel('Electron momentum y-direction')
plt.ylabel('count')
plt.legend(loc='upper right')
#plt.xlim((0, 0.25))
plt.ylim((0, 12))
plt.show()
p3e_nn = prediction[:,2] 
p3e = y[:,2]
print(p3e_nn)
print(p3e)
plt.hist(p3e_nn, 100, alpha=0.5, label='NN prediction', density = True)
plt.hist(p3e, 100, alpha=0.5, label='Photon Momentum from simulations', density = True)
plt.xlabel('Electron momentum z-direction')
plt.ylabel('count')
plt.legend(loc='upper right')
plt.ylim((0, 12))
plt.show()
fig, axs = plt.subplots(3, sharex = True)
fig.suptitle('3-momentum electrons post interaction')
axs[0].hist(p1e_nn, 100, alpha=0.5, label='NN prediction', density = True)
axs[0].hist(p1e, 100, alpha=0.5, label='NN prediction', density = True)
axs[1].hist(p2e_nn, 100, alpha=0.5, label='Photon Momentum from simulations', density = True)
axs[1].hist(p2e, 100, alpha=0.5, label='Photon Momentum from simulations', density = True)
axs[2].hist(p3e_nn, 100, alpha=0.5, label='Photon Momentum from simulations', density = True)
axs[2].hist(p3e, 100, alpha=0.5, label='Photon Momentum from simulations', density = True)
fig.show()
final_e = []
final_e_nn = []
for classi_ in class_data:
    final_e.append(np.linalg.norm(classi_[-7:-4]))
from tensorflow import keras 
final_e_nn = []
threshold = 0.5
num = int(1*len(x))
bar = progressbar.ProgressBar(maxval=num, 
                              widgets=[progressbar.Bar('=', '[', ']'), ' ', 
                                       progressbar.Percentage(), 
                                       " of {0}".format(num)])
bar.start()
for pred in prediction:
    final_e_nn.append(np.linalg.norm(pred))
bar.finish()
energy_spectrum(final_e, 75)
energy_spectrum(final_e_nn, 75)
from scipy.stats import norm
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import scipy.stats as stats
from scipy.stats import chisquare

mean,std=norm.fit(final_e)
plt.hist(final_e, bins=100, alpha = 0.5, label='NN prediction', density = True)
xmin, xmax = plt.xlim()
x = np.linspace(xmin, xmax, 100)
y = norm.pdf(x, mean, std)
plt.plot(x, y,'r--', linewidth=2)
plt.legend(loc='upper right')
plt.show()
print('mean = ', mean)
print('std = ', std)
print("chi square = ", stats.chisquare(final_e))
from scipy.stats import norm
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import scipy.stats as stats
from scipy.stats import chisquare

mean_nn,std_nn=norm.fit(final_e_nn)
plt.hist(final_e_nn, bins=100, alpha = 0.5, label='NN prediction', density = True)
xmin, xmax = plt.xlim()
x = np.linspace(xmin, xmax, 100)
y = norm.pdf(x, mean_nn, std_nn)
plt.plot(x, y,'r--', linewidth=2)
plt.legend(loc='upper right')
plt.show()
print('mean = ', mean_nn)
print('std = ', std_nn)
print("chi square = ", stats.chisquare(final_e_nn))
plt.hist(final_e_nn, bins=100, alpha = 0.5, label='NN prediction', density = True)
plt.hist(final_e, bins=100, alpha = 0.5, label='Electron Momentum from simulations', density = True)
x_nn = np.linspace(xmin, xmax, 100)
y_nn = norm.pdf(x_nn, mean_nn, std_nn)
plt.plot(x_nn, y_nn,'r--', label = 'fit NN', linewidth = 2)
plt.legend(loc='upper right')
x_e = np.linspace(xmin, xmax, 100)
y_e = norm.pdf(x_e, mean, std)
plt.plot(x_e, y_e, 'g:', label = 'fit Electron Momentum Simulations', linewidth = 2)
plt.legend(loc = 'upper right')
plt.ylim((0, 30))
plt.savefig('Fit_250train_250test.png')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15702150252190072, 'fraction_numerical': 0.0415449960180515, 'mean_word_length': 4.8140432098765435, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '37506', 'license': 'MIT', 'n_tokens_mistral': 3115, 'n_tokens_neox': 2868, 'n_words': 657, 'path': 'notebooks/.ipynb_checkpoints/Comp_Scatt_NeuralNetwork_ClassificationandRegression-checkpoint.ipynb', 'repo_name': 'chiarabadiali/comp_scatt_ML', 'size': '7534'}