<jupyter_start><jupyter_text>Print DependenciesDependences are fundamental to record the computational environment.<jupyter_code>%load_ext watermark

# python, ipython, packages, and machine characteristics
%watermark -v -m -p pandas,keras,numpy,math,tensorflow,matplotlib,h5py

# date
print (" ")
%watermark -u -n -t -z<jupyter_output>Python implementation: CPython
Python version       : 3.9.6
IPython version      : 7.23.1

pandas    : 1.2.3
keras     : 2.4.3
numpy     : 1.19.5
math      : unknown
tensorflow: 2.5.0
matplotlib: 3.4.2
h5py      : 3.1.0

Compiler    : Clang 12.0.0 (clang-1200.0.32.29)
OS          : Darwin
Release     : 19.6.0
Machine     : x86_64
Processor   : i386
CPU cores   : 8
Architecture: 64bit

 
Last updated: Fri Jul 23 2021 16:40:39WEST<jupyter_text>Some useful functions<jupyter_code>def prepare_data(class_data): 
    np.random.shuffle(class_data)
    y = class_data_low[:,0]
    x = class_data_low[:,1:10]
    train_split = 0.85
    train_limit = int(len(y)*train_split)
    print("Training sample of: {0} \nValuation sample: {1}".format(train_limit, len(y)-train_limit))
    x_train = x[:train_limit]
    x_val = x[train_limit:]
    y_train = y[:train_limit]
    y_val = y[train_limit:]
    if np.isnan(np.min(y)) == False:
        return x_train, y_train, x_val, y_val
def plot_histo(x, y, bins,logscale):
    y = np.array(y)
    pyplot.hist(y, bins, color = 'indianred', alpha=0.5, label='Osiris')
    pyplot.legend(loc='upper right')
    pyplot.xlabel('Probability')
    if logscale == 1:
        pyplot.yscale('log')
    pyplot.title('Trained on ($p_e$, $p_{\gamma}$, $\omega_e$, $\omega_{\gamma}$, n)')
    pyplot.show()
def loaddata(name):
     return np.loadtxt(name, delimiter=',')
def balance_data(class_data, nbins):

    from matplotlib import pyplot as plt
    y = class_data[:,0]
    n, edges, _ = plt.hist(y, nbins, color = 'indianred', alpha=0.5, label='Osiris')
    n_max = int(n.max())
    data = []
    bar = progressbar.ProgressBar(maxval=len(class_data), 
                              widgets=[progressbar.Bar('=', '[', ']'), ' ', 
                                       progressbar.Percentage(), 
                                       " of {0}".format(len(class_data))])
    bar.start()
    for k, class_ in enumerate(class_data):
        for i in range(len(n)):
            edges_min = edges[i]
            edges_max = edges[i+1]
            if class_[0] > edges_min and class_[0] < edges_max:
                for j in range(int(n_max/(n[i]))):
                    data.append(class_)
                break
        bar.update(k+1)
    bar.finish()

    return np.array(data)
def plot_history(history):
    loss = history.history['loss']
    val_loss = history.history['val_loss']

    accuracy = history.history['mape']
    val_accuracy = history.history['val_mape']


    epochs = range(1, len(loss) + 1)
    fig, ax1 = plt.subplots()

    l1 = ax1.plot(epochs, loss, 'bo', label='Training loss')
    vl1 = ax1.plot(epochs, val_loss, 'b', label='Validation loss')
    ax1.set_title('Training and validation loss')
    ax1.set_xlabel('Epochs')
    ax1.set_ylabel('Loss (mae))')

    ax2 = ax1.twinx()
    ac2= ax2.plot(epochs, accuracy, 'o', c="red", label='Training acc')
    vac2= ax2.plot(epochs, val_accuracy, 'r', label='Validation acc')
    ax2.set_ylabel('mape')

    lns = l1 + vl1 + ac2 + vac2
    labs = [l.get_label() for l in lns]
    ax2.legend(lns, labs, loc="center right")
    fig.tight_layout()
    fig.show()<jupyter_output><empty_output><jupyter_text>Data-set loading<jupyter_code>class_data_low = loaddata("../final_data/{}.csv".format('0-10'))
class_data_high = loaddata("../final_data/{}.csv".format('0-50'))
class_data_shigh = loaddata("../final_data/{}.csv".format('0-100'))<jupyter_output><empty_output><jupyter_text>Our beautiful probability distributions (non balanced)<jupyter_code>x0_train, y0_train, x0_val, y0_val = prepare_data(class_data_low)
x1_train, y1_train, x1_val, y1_val = prepare_data(class_data_high)
x2_train, y2_train, x2_val, y2_val = prepare_data(class_data_shigh)
bins = np.linspace(0.0001, 0.015, 100)
plot_histo(x0_train,y0_train,bins,0)
bins = np.linspace(0.0001, 0.015, 100)
plot_histo(x1_train,y1_train,bins,0)
bins = np.linspace(0.0001, 0.015, 100)
plot_histo(x2_train,y2_train,bins,0)<jupyter_output><empty_output><jupyter_text>Balancing of the data<jupyter_code>class_data_low_b = balance_data(class_data_low, 100)
class_data_high_b = balance_data(class_data_high, 100)
class_data_shigh_b = balance_data(class_data_shigh, 100)
x0_train, y0_train, x0_val, y0_val = prepare_data(class_data_low_b)
x1_train, y1_train, x1_val, y1_val = prepare_data(class_data_high_b)
x2_train, y2_train, x2_val, y2_val = prepare_data(class_data_shigh_b)<jupyter_output><empty_output><jupyter_text>Model Build__1)__ This is the first step in our chaing of Transfer Learning. We have to buil the model that we're going to reuse also in the next trainings.<jupyter_code>from keras.models import Sequential
from keras.layers.core import Dense
import keras.backend as K
from keras import optimizers
from keras import models
from keras import layers
from keras.layers.normalization import BatchNormalization
def build_model() :
    model = models.Sequential()
    model.add (BatchNormalization(input_dim = 9))
    model.add (layers.Dense (12 , activation = "sigmoid"))
    model.add (layers.Dense (9 , activation = "relu"))
    model.add (layers.Dense (1 , activation = "sigmoid"))
    model.compile(optimizer = "adam" , loss = 'mae' , metrics = ["mape", tf.keras.metrics.KLDivergence()])
    return model
model = build_model ()
history0 = model.fit ( x_train0, y_train0, epochs = 5000, batch_size = 1000 , validation_data = (x_val0, y_val0) )
model.save("../models/{}.h5".format('transferlearning1'))
model.summary()
plot_history(history0)
history1 = model.fit ( x_train1, y_train1, epochs = 1000, batch_size = 1000, validation_data = (x_val1, y_val1) )
model.save("../models/{}.h5".format('transferlearning2'))
plot_history(history1)
history2 = model.fit ( x_train2, y_train2, epochs = 1000, batch_size = 1000, validation_data = (x_val2, y_val2) )
model.save("../models/{}.h5".format('transferlearning3'))
plot_history(history2)<jupyter_output><empty_output><jupyter_text>Probability density distribution<jupyter_code>x0_train, y0_train, x0_val, y0_val = prepare_data(class_data_low)
x1_train, y1_train, x1_val, y1_val = prepare_data(class_data_high)
x2_train, y2_train, x2_val, y2_val = prepare_data(class_data_shigh)
x_train = np.vstack((x0_train,x1_train,x2_train))
x_val = np.vstack((x0_val,x1_val,x2_val))
y_train = np.vstack((y0_train,y1_train,y2_train))
y_val = np.vstack((y0_val,y1_val,y2_val))
y_pred0 = model.predict(x0_val)
from matplotlib import pyplot
y0_val = np.array(y0_val)
bins = np.linspace(0.0004, 0.6, 60)
pyplot.hist(y0_val, bins, color = 'indianred', alpha=0.5, label='Osiris')
pyplot.hist(y_pred0, bins, color = 'mediumslateblue', alpha=0.5, label='NN')
#pyplot.yscale('log')
pyplot.legend(loc='upper right')
pyplot.xlabel('Probability')
pyplot.title('Trained on ($p_e$, $p_{\gamma}$, $\omega_e$, $\omega_{\gamma}$, n)')
pyplot.show()
y_pred1 = model.predict(x1_val)
from matplotlib import pyplot
y1_val = np.array(y1_val)
bins = np.linspace(0.001, 0.013, 100)
pyplot.hist(y1_val, bins, color = 'indianred', alpha=0.5, label='Osiris')
pyplot.hist(y_pred1, bins, color = 'mediumslateblue', alpha=0.5, label='NN')
#pyplot.yscale('log')
pyplot.legend(loc='upper right')
pyplot.xlabel('Probability')
pyplot.title('Trained on ($p_e$, $p_{\gamma}$, $\omega_e$, $\omega_{\gamma}$, n)')
pyplot.show()
y_pred2 = model.predict(x2_val)
from matplotlib import pyplot
y2_val = np.array(y2_val)
bins = np.linspace(0.0004, 0.6, 60)
pyplot.hist(y2_val, bins, color = 'indianred', alpha=0.5, label='Osiris')
pyplot.hist(y_pred2, bins, color = 'mediumslateblue', alpha=0.5, label='NN')
#pyplot.yscale('log')
pyplot.legend(loc='upper right')
pyplot.xlabel('Probability')
pyplot.title('Trained on ($p_e$, $p_{\gamma}$, $\omega_e$, $\omega_{\gamma}$, n)')
pyplot.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14914123316446312, 'fraction_numerical': 0.041270233535153834, 'mean_word_length': 4.491180461329715, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '432652', 'license': 'MIT', 'n_tokens_mistral': 3286, 'n_tokens_neox': 3098, 'n_words': 595, 'path': 'final_notebook/.ipynb_checkpoints/Comp_Scatt_NeuralNetwork-checkpoint.ipynb', 'repo_name': 'chiarabadiali/comp_scatt_ML', 'size': '8093'}