<commit_before>import os
from keras.models import Sequential, Model, load_model
from keras.layers import Dense, Activation, Input, LSTM, Masking, Dropout
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
from array import array
from MLHelper import Root2NumpyConverter, TrainingReader
from PyAnalysisTools.base import _logger
from PyAnalysisTools.base.ShellUtils import make_dirs
from PyAnalysisTools.ROOTUtils.FileHandle import FileHandle


class NeuralNetwork(object):
    def __init__(self, num_features, num_layers=3, size=20, lr=1e-3, keep_prob=1.0, tloss="soft", input_noise=0.0):
        self.inputs = inputs = Input(shape=(num_features,))
        x = Dense(size, activation='relu')(inputs)
        for i in range(num_layers - 1):
            x = Dense(size, activation='tanh')(x)
        pred = Dense(1, activation='sigmoid')(x)
        #pred = Dense(1, activation="softmax")(x)
        model = Model(inputs=inputs, outputs=pred)

        # self.train_op = Adam(lr)
        model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
        self.kerasmodel = model


class NNTrainer(object):
    def __init__(self, **kwargs):
        kwargs.setdefault("n_features", None)
        kwargs.setdefault("layers", 2)
        kwargs.setdefault("units", 10)
        kwargs.setdefault("epochs", 10)
        self.reader = TrainingReader(**kwargs)
        self.converter = Root2NumpyConverter(kwargs["variables"])
        self.n_features = kwargs["n_features"]
        self.layers = kwargs["layers"]
        self.units = kwargs["units"]
        self.epochs = kwargs["epochs"]
        self.plot = True
        self.output_path = kwargs["output_path"]
        make_dirs(os.path.join(self.output_path, "plots"))
        make_dirs(os.path.join(self.output_path, "models"))

    def build_input(self):
        trees = self.reader.get_trees()
        arrays = [[self.converter.convert_to_array(tree) for tree in items]
                  for items in trees]
        self.data_train, self.label_train = self.converter.merge(arrays[0], arrays[1])
        self.data_eval, self.label_eval = self.converter.merge(arrays[2], arrays[3])

    def build_models(self):
        if self.n_features is None:
            self.n_features = self.data_train.shape[1]
        self.model_0 = NeuralNetwork(self.n_features, num_layers=self.layers, size=self.units, tloss='soft').kerasmodel
        self.model_1 = NeuralNetwork(self.n_features, num_layers=self.layers, size=self.units, tloss='soft').kerasmodel

    def apply_scaling(self):
        train_mean = self.data_train.mean(0)
        train_std = self.data_train.std()
        self.data_train = (self.data_train - train_mean) / train_std
        eval_mean = self.data_eval.mean(0)
        eval_std = self.data_eval.std()
        self.data_eval = (self.data_eval - eval_mean) / eval_std

    def plot_train_control(self, history, name):
        plt.plot(history.history['loss'])
        plt.plot(history.history['val_loss'])
        plt.plot(history.history['acc'])
        plt.plot(history.history['val_acc'])
        plt.legend(["loss", "valid loss", "acc", "valid acc"])
        plt.xlabel('epoch')
        plt.ylabel('NN loss')
        plt.savefig(os.path.join(self.output_path, 'plots/{:s}.png'.format(name)))
        plt.close()

    def train(self):
        self.build_input()
        self.build_models()
        self.apply_scaling()
        history_train = self.model_0.fit(self.data_train, self.label_train, epochs=self.epochs, verbose=1, batch_size=32,
                                         shuffle=True, validation_data=(self.data_eval, self.label_eval))  # sample_weight=weight_0,
        history_eval = self.model_1.fit(self.data_eval, self.label_eval, epochs=self.epochs, verbose=1, batch_size=32,
                                        shuffle=True, validation_data=(self.data_train, self.label_train)) #sample_weight=weight_0,
        if self.plot:
            self.plot_train_control(history_train, "train")
            self.plot_train_control(history_eval, "eval")
        _logger.debug("Done Training, saving models...")

        self.model_0.save(os.path.join(self.output_path, "models/model_train.h5"))
        self.model_1.save(os.path.join(self.output_path, "models/model_eval.h5"))
        self.run_predictions()

    def run_predictions(self):
        _logger.info("Evaluating predictions")
        preds_train = self.model_0.predict(self.data_eval)
        preds_sig_train = preds_train[self.label_eval == 1]
        preds_bkg_train = preds_train[self.label_eval == 0]
        if self.plot:
            _logger.debug("Consistency plots")
            plt.hist(preds_sig_train, 20, range=[0., 1.], histtype='step', label='signal model0', normed=True)
            plt.hist(preds_bkg_train, 20, range=[0., 1.], histtype='step', label='bkg model1', normed=True)
            plt.yscale('log')
            plt.grid(True)
            plt.legend(["signal", "background"], loc="lower left")
            plt.savefig(os.path.join(self.output_path, "plots/consistency_sig_train.png"))


class NNReader(object):
    def __init__(self, **kwargs):<commit_msg>NN updates<commit_after>import os
from keras.models import Sequential, Model, load_model
from keras.layers import Dense, Activation, Input, LSTM, Masking, Dropout, Flatten, Reshape
from keras.layers.normalization import BatchNormalization
from keras.optimizers import SGD

import keras.backend as backend
from keras.optimizers import Adam
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from array import array
from MLHelper import Root2NumpyConverter, TrainingReader
from PyAnalysisTools.base import _logger
from PyAnalysisTools.base.ShellUtils import make_dirs
from PyAnalysisTools.ROOTUtils.FileHandle import FileHandle
from PyAnalysisTools.base.OutputHandle import SysOutputHandle as so


class NeuralNetwork(object):
    def __init__(self, num_features, num_layers=3, size=20, lr=1e-3, keep_prob=1.0, tloss="soft", input_noise=0.0):
        # self.inputs = inputs = Input(shape=num_features)
        # print backend.int_shape(inputs)
        # x = Reshape((-1,))(inputs)
        # print backend.int_shape(x)
        #
        # x = Dense(size, activation='relu')(x)
        # for i in range(num_layers - 1):
        #     x = Dense(size, activation='tanh')(x)
        # #pred = Dense(1, activation='softmax')(x)
        # pred = Dense(1, activation="sigmoid")(x)
        # model = Model(inputs=inputs, outputs=pred)
        # # self.train_op = Adam(lr)
        # model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
        # self.kerasmodel = model

        self.inputs = inputs = Input(shape=num_features)
        # print backend.int_shape(inputs)
        # x = Reshape((-1,))(inputs)
        # print backend.int_shape(x)
        model =Sequential()
        #model.add(Dense(num_features[0], input_shape=(num_features[1],)))
        model.add(Dense(64, input_dim=num_features[1], activation='relu'))
        model.add(Dense(64, activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(64, activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(1, activation='sigmoid'))
        sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)

        # model.add(Activation('relu'))
        # x = Dense(size, activation='relu')(x)
        # for i in range(num_layers - 1):
        #     x = Dense(size, activation='tanh')(x)
        # #pred = Dense(1, activation='softmax')(x)
        # pred = Dense(1, activation="sigmoid")(x)
        # model = Model(inputs=inputs, outputs=pred)
        # self.train_op = Adam(lr)
        model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
        self.kerasmodel = model


class NNTrainer(object):
    def __init__(self, **kwargs):
        kwargs.setdefault("n_features", None)
        kwargs.setdefault("layers", 2)
        kwargs.setdefault("units", 10)
        kwargs.setdefault("epochs", 10)
        self.reader = TrainingReader(**kwargs)
        self.converter = Root2NumpyConverter(kwargs["variables"])
        self.n_features = kwargs["n_features"]
        self.layers = kwargs["layers"]
        self.units = kwargs["units"]
        self.epochs = kwargs["epochs"]
        self.max_events = kwargs["max_events"]
        self.plot = True
        self.output_path = so.resolve_output_dir(output_dir=kwargs["output_path"], sub_dir_name="NNtrain")
        make_dirs(os.path.join(self.output_path, "plots"))
        make_dirs(os.path.join(self.output_path, "models"))

    def build_input(self):
        trees = self.reader.get_trees()
        arrays = [[self.converter.convert_to_array(tree, max_events=self.max_events) for tree in items]
                  for items in trees]
        self.data_train, self.label_train = self.converter.merge(arrays[0], arrays[1])
        self.data_eval, self.label_eval = self.converter.merge(arrays[2], arrays[3])
        self.data_train = pd.DataFrame(self.data_train)
        self.data_eval = pd.DataFrame(self.data_eval)

    def build_models(self):
        self.model_0 = NeuralNetwork(self.data_train.shape, num_layers=self.layers, size=self.units,
                                     tloss='soft').kerasmodel
        self.model_1 = NeuralNetwork(self.data_eval.shape, num_layers=self.layers, size=self.units,
                                     tloss='soft').kerasmodel

    def apply_scaling(self):
        train_mean = self.data_train.mean()
        train_std = self.data_train.std()
        self.data_train = (self.data_train - train_mean) / train_std
        eval_mean = self.data_eval.mean()
        eval_std = self.data_eval.std()
        self.data_eval = (self.data_eval - eval_mean) / eval_std

    def plot_train_control(self, history, name):
        plt.plot(history.history['loss'])
        plt.plot(history.history['val_loss'])
        plt.plot(history.history['acc'])
        plt.plot(history.history['val_acc'])
        plt.legend(["loss", "valid loss", "acc", "valid acc"])
        plt.xlabel('epoch')
        plt.ylabel('NN loss')
        plt.savefig(os.path.join(self.output_path, 'plots/{:s}.png'.format(name)))
        plt.close()

    def train(self):
        self.build_input()
        self.build_models()
        self.apply_scaling()
        #print "train shape: ", train_shape
        history_train = self.model_0.fit(self.data_train.values, self.label_train.reshape((self.label_train.shape[0], 1)),
                                         epochs=self.epochs, verbose=1, batch_size=32,
                                         shuffle=True, validation_data=(self.data_eval.values, self.label_eval))  # sample_weight=weight_0,
        history_eval = self.model_1.fit(self.data_eval.values, self.label_eval.reshape((self.label_eval.shape[0], 1)), epochs=self.epochs,
                                        verbose=1, batch_size=32,
                                        shuffle=True, validation_data=(self.data_train.values, self.label_train)) #sample_weight=weight_0,
        if self.plot:
            self.plot_train_control(history_train, "train")
            self.plot_train_control(history_eval, "eval")
        _logger.debug("Done Training, saving models...")

        self.model_0.save(os.path.join(self.output_path, "models/model_train.h5"))
        self.model_1.save(os.path.join(self.output_path, "models/model_eval.h5"))
        self.run_predictions()

    def run_predictions(self):
        _logger.info("Evaluating predictions")
        preds_train = self.model_0.predict(self.data_eval.values)
        preds_sig_train = preds_train[self.label_eval == 1]
        preds_bkg_train = preds_train[self.label_eval == 0]
        if self.plot:
            _logger.debug("Consistency plots")
            plt.hist(preds_sig_train, 20, range=[0., 1.], histtype='step', label='signal model0', normed=True)
            plt.hist(preds_bkg_train, 20, range=[0., 1.], histtype='step', label='bkg model1', normed=True)
            plt.yscale('log')
            plt.grid(True)
            plt.legend(["signal", "background"], loc="lower left")
            plt.savefig(os.path.join(self.output_path, "plots/consistency_sig_train.png"))


class NNReader(object):
    def __init__(self, **kwargs):
{'fraction_non_alphanumeric': 0.13110001610565308, 'fraction_numerical': 0.011032372362699307, 'mean_word_length': 3.0203949498219487, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '930430', 'n_tokens_mistral': 4011, 'n_tokens_neox': 3907, 'n_words': 679}