# Path: minos/experiment/training.py
# class EpochStoppingCondition(object):
#     """ Stop training after a fixed number of epochs
#         # Arguments
#         epoch: number of epochs after which training will be stopped.
#     """
# 
#     def __init__(self, epoch):
#         self.epoch = epoch
# 
#     def todict(self):
#         return dict(vars(self))
# 
# class AccuracyDecreaseStoppingCondition(object):
#     """ Stop training when the metric observed has stopped improving.
#         # Arguments
#         noprogress_count: number of epochs with no improvement
#             after which training will be stopped.
#         min_epoch: minimum number of epochs.
#         max_epoch: maximum number of epochs.
#     """
# 
#     def __init__(self, max_epoch, metric='accuracy',
#                  validation_metric=True, noprogress_count=3, min_epoch=0):
#         self.metric = metric
#         self.validation_metric = validation_metric
#         self.noprogress_count = noprogress_count
#         self.min_epoch = min_epoch
#         self.max_epoch = max_epoch
#         self.epoch = 0
# 
#     def get_monitor_metric(self):
#         if self.validation_metric:
#             return get_associated_validation_metric(self.metric)
#         return self.metric
# 
#     def is_min_epoch_defined(self):
#         return self.min_epoch and self.min_epoch > 0
# 
#     def is_max_epoch_defined(self):
#         return self.max_epoch and self.max_epoch > 0
# 
#     def is_at_least_min_epoch(self):
#         return self.is_min_epoch_defined()\
#             and self.epoch >= self.min_epoch
# 
#     def is_at_most_max_epoch(self):
#         return self.is_max_epoch_defined()\
#             and self.epoch <= self.max_epoch
# 
#     def todict(self):
#         return dict(vars(self))
# 
# class StoppingConditionWrapper(EarlyStopping):
# 
#     def __init__(self, condition):
#         super().__init__(
#             monitor=condition.get_monitor_metric(),
#             patience=condition.noprogress_count)
#         self.condition = condition
# 
#     def on_epoch_end(self, epoch, logs=None):
#         self.condition.epoch = epoch
#         if self.condition.is_min_epoch_defined()\
#                 and not self.condition.is_at_least_min_epoch():
#             return
#         if self.condition.is_max_epoch_defined()\
#                 and not self.condition.is_at_most_max_epoch():
#             self.stopped_epoch = epoch
#             self.model.stop_training = True
#             return
#         super().on_epoch_end(epoch, logs)
# 
# def get_associated_validation_metric(metric):
#     if not metric:
#         return None
#     if metric.startswith('val_'):
#         return metric
#     return 'val_%s' % metric
# 
# Path: minos/tf_utils.py
# def setup_tf_session(device):
#     import tensorflow as tf
#     config = tf.ConfigProto()
#     if hasattr(config, 'gpu_options'):
#         config.allow_soft_placement = True
#         config.gpu_options.allow_growth = True
#     if is_gpu_device(device):
#         config.gpu_options.visible_device_list = str(
#             get_device_idx(device))
#     elif is_cpu_device(device):
#         config.gpu_options.visible_device_list = ''
#     from keras import backend
#     session = tf.Session(config=config)
#     session.run(tf.global_variables_initializer())
#     backend.set_session(session)
# 
# Path: minos/utils.py
# def disable_sysout():
#     sys.stdout.write = lambda s: s
# 
# def load_keras_model(filename, device=None):
#     setup_tf_session(device)
#     custom_objects = dict()
#     custom_objects.update(get_custom_activations())
#     custom_objects.update({
#         layer[0].__name__: layer[0]
#         for _name, layer in get_custom_layers().items()})
#     return load_model(filename, custom_objects=custom_objects)


import logging
import traceback
import numpy
from multiprocessing import Queue, Process
from threading import Thread
from time import time
from keras.callbacks import ModelCheckpoint
from minos.experiment.training import EpochStoppingCondition,\
    AccuracyDecreaseStoppingCondition, StoppingConditionWrapper,\
    get_associated_validation_metric
from minos.tf_utils import setup_tf_session
from minos.utils import disable_sysout, load_keras_model
        from minos.model.build import ModelBuilder
            from keras import backend

    def __init__(self, batch_iterator, test_batch_iterator):
        self.model_builder = ModelBuilder()
        self.batch_iterator = batch_iterator
        self.test_batch_iterator = test_batch_iterator

    def train(self, blueprint, device,
              save_best_model=False, model_filename=None, class_weight=None):
        try:
            model = self.model_builder.build(
                blueprint,
                device)
            class_weight = class_weight or blueprint.training.class_weight
            setup_tf_session(device)
            nb_epoch, callbacks = self._get_stopping_parameters(blueprint)
            if save_best_model:
                callbacks.append(self._get_model_save_callback(
                    model_filename,
                    blueprint.training.metric.metric))
            start = time()
            history = model.fit_generator(
                self.batch_iterator,
                self.batch_iterator.samples_per_epoch,
                nb_epoch,
                callbacks=callbacks,
                validation_data=self.test_batch_iterator,
                nb_val_samples=self.test_batch_iterator.sample_count,
                class_weight=class_weight)
            if save_best_model:
                del model

# Path: minos/experiment/training.py
# class EpochStoppingCondition(object):
#     """ Stop training after a fixed number of epochs
#         # Arguments
#         epoch: number of epochs after which training will be stopped.
#     """
# 
#     def __init__(self, epoch):
#         self.epoch = epoch
# 
#     def todict(self):
#         return dict(vars(self))
# 
# class AccuracyDecreaseStoppingCondition(object):
#     """ Stop training when the metric observed has stopped improving.
#         # Arguments
#         noprogress_count: number of epochs with no improvement
#             after which training will be stopped.
#         min_epoch: minimum number of epochs.
#         max_epoch: maximum number of epochs.
#     """
# 
#     def __init__(self, max_epoch, metric='accuracy',
#                  validation_metric=True, noprogress_count=3, min_epoch=0):
#         self.metric = metric
#         self.validation_metric = validation_metric
#         self.noprogress_count = noprogress_count
#         self.min_epoch = min_epoch
#         self.max_epoch = max_epoch
#         self.epoch = 0
# 
#     def get_monitor_metric(self):
#         if self.validation_metric:
#             return get_associated_validation_metric(self.metric)
#         return self.metric
# 
#     def is_min_epoch_defined(self):
#         return self.min_epoch and self.min_epoch > 0
# 
#     def is_max_epoch_defined(self):
#         return self.max_epoch and self.max_epoch > 0
# 
#     def is_at_least_min_epoch(self):
#         return self.is_min_epoch_defined()\
#             and self.epoch >= self.min_epoch
# 
#     def is_at_most_max_epoch(self):
#         return self.is_max_epoch_defined()\
#             and self.epoch <= self.max_epoch
# 
#     def todict(self):
#         return dict(vars(self))
# 
# class StoppingConditionWrapper(EarlyStopping):
# 
#     def __init__(self, condition):
#         super().__init__(
#             monitor=condition.get_monitor_metric(),
#             patience=condition.noprogress_count)
#         self.condition = condition
# 
#     def on_epoch_end(self, epoch, logs=None):
#         self.condition.epoch = epoch
#         if self.condition.is_min_epoch_defined()\
#                 and not self.condition.is_at_least_min_epoch():
#             return
#         if self.condition.is_max_epoch_defined()\
#                 and not self.condition.is_at_most_max_epoch():
#             self.stopped_epoch = epoch
#             self.model.stop_training = True
#             return
#         super().on_epoch_end(epoch, logs)
# 
# def get_associated_validation_metric(metric):
#     if not metric:
#         return None
#     if metric.startswith('val_'):
#         return metric
#     return 'val_%s' % metric
# 
# Path: minos/tf_utils.py
# def setup_tf_session(device):
#     import tensorflow as tf
#     config = tf.ConfigProto()
#     if hasattr(config, 'gpu_options'):
#         config.allow_soft_placement = True
#         config.gpu_options.allow_growth = True
#     if is_gpu_device(device):
#         config.gpu_options.visible_device_list = str(
#             get_device_idx(device))
#     elif is_cpu_device(device):
#         config.gpu_options.visible_device_list = ''
#     from keras import backend
#     session = tf.Session(config=config)
#     session.run(tf.global_variables_initializer())
#     backend.set_session(session)
# 
# Path: minos/utils.py
# def disable_sysout():
#     sys.stdout.write = lambda s: s
# 
# def load_keras_model(filename, device=None):
#     setup_tf_session(device)
#     custom_objects = dict()
#     custom_objects.update(get_custom_activations())
#     custom_objects.update({
#         layer[0].__name__: layer[0]
#         for _name, layer in get_custom_layers().items()})
#     return load_model(filename, custom_objects=custom_objects)


# Path: minos/train/trainer.py
import logging
import traceback
import numpy
from multiprocessing import Queue, Process
from threading import Thread
from time import time
from keras.callbacks import ModelCheckpoint
from minos.experiment.training import EpochStoppingCondition,\
    AccuracyDecreaseStoppingCondition, StoppingConditionWrapper,\
    get_associated_validation_metric
from minos.tf_utils import setup_tf_session
from minos.utils import disable_sysout, load_keras_model
        from minos.model.build import ModelBuilder
            from keras import backend


    def __init__(self, batch_iterator, test_batch_iterator):
        self.model_builder = ModelBuilder()
        self.batch_iterator = batch_iterator
        self.test_batch_iterator = test_batch_iterator

    def train(self, blueprint, device,
              save_best_model=False, model_filename=None, class_weight=None):
        try:
            model = self.model_builder.build(
                blueprint,
                device)
            class_weight = class_weight or blueprint.training.class_weight
            setup_tf_session(device)
            nb_epoch, callbacks = self._get_stopping_parameters(blueprint)
            if save_best_model:
                callbacks.append(self._get_model_save_callback(
                    model_filename,
                    blueprint.training.metric.metric))
            start = time()
            history = model.fit_generator(
                self.batch_iterator,
                self.batch_iterator.samples_per_epoch,
                nb_epoch,
                callbacks=callbacks,
                validation_data=self.test_batch_iterator,
                nb_val_samples=self.test_batch_iterator.sample_count,
                class_weight=class_weight)
            if save_best_model:
                del model

                model = load_keras_model(model_filename)