<commit_before># -*- coding: utf-8 -*-
from __future__ import division
from __future__ import absolute_import

import os
import time

from keras.optimizers import Adadelta, Adam, Adamax, Adagrad, RMSprop, SGD

from aetros.backend import JobImage

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO
from io import BytesIO

import PIL.Image
import math
from keras.callbacks import Callback
from keras import backend as K
import keras.layers.convolutional

import numpy as np

from aetros.utils.image import get_layer_vis_square, get_layer_vis_square_raw, get_image_tales
from .keras_model_utils import ensure_dir, get_total_params
import six

class KerasLogger(Callback):
    def __init__(self, trainer, job_backend, general_logger, force_insights=False):
        self.params = {}
        super(KerasLogger, self).__init__()
        self.validation_per_batch = []
        self.insight_layer = []
        self.ins = None
        self.insights_sample_path = None
        self.force_insights = force_insights

        self.trainer = trainer
        self.current = {}
        self.log_epoch = True
        self.confusion_matrix = True

        self.job_backend = job_backend
        self.job_model = job_backend.get_job_model()
        self.general_logger = general_logger
        self._test_with_acc = None
        self.last_batch_time = time.time()
        self.start_time = time.time()
        self.made_batches = 0
        self.batches_per_second = 0
        self.stats = []
        self.last_current = None
        self.filepath_best = self.job_model.get_weights_filepath_best()
        self.filepath_latest = self.job_model.get_weights_filepath_latest()

        ensure_dir(os.path.dirname(self.filepath_best))
        self.data_gathered = False

        self.accuracy_channel = None
        self.all_losses = None
        self.loss_channel = None
        self.learning_rate_channel = None

        self.learning_rate_start = 0

        self.insights_x = None
        self.best_total_accuracy = 0

    def on_train_begin(self, logs={}):
        self.start_time = time.time()
        self.last_batch_time = time.time()
        self.job_backend.set_status('TRAINING')
        self.job_backend.set_system_info('total_params', get_total_params(self.model))

        if self.model.optimizer and hasattr(self.model.optimizer, 'get_config'):
            config = self.model.optimizer.get_config()
            self.job_backend.set_info('optimizer', type(self.model.optimizer).__name__)
            for i, v in config.items():
                self.job_backend.set_info('optimizer.'+str(i), v)

        self.current['epoch'] = 0
        self.current['started'] = self.start_time
        self.job_backend.set_system_info('current', self.current)

        xaxis = {
            'range': [1, self.params['epochs']],
            'title': u'Epoch ⇢'
        }
        yaxis = {
            'range': [1, 100],
            'title': u'% Accuracy ⇢',
            'dtick': 10
        }

        traces = ['validation', 'training']
        if len(self.model.output_layers) > 1:
            traces = []
            for output in self.model.output_layers:
                traces.append(output.name+'_validation')
                traces.append(output.name+'_training')

        self.accuracy_channel = self.job_backend.create_channel('accuracy', main=True, traces=traces, kpi=True, max_optimization=True, xaxis=xaxis, yaxis=yaxis)
        self.loss_channel = self.job_backend.create_loss_channel('loss', xaxis=xaxis)
        self.learning_rate_channel = self.job_backend.create_channel('learning rate', traces=['start', 'end'], xaxis=xaxis)
        self.job_backend.progress(0, self.params['epochs'])
        if len(self.model.output_layers) > 1:
            loss_traces = []
            for output in self.model.output_layers:
                loss_traces.append(output.name+'_validation')
                loss_traces.append(output.name+'_training')

            self.all_losses = self.job_backend.create_channel('All loss', main=True, xaxis=xaxis, traces=loss_traces)

        images = self.build_insight_images()
        self.job_backend.job_add_insight(0, images, None)

    def on_batch_begin(self, batch, logs={}):

        if not self.data_gathered:
            validation_samples = 0
            if 'samples' in self.params:
                training_samples = self.params['samples']
                validation_samples = len(self.model.validation_data[0]) if self.model.validation_data else self.trainer.nb_val_samples
            else:
                training_samples = self.params['steps'] * logs['size']
                if self.trainer.is_generator(self.trainer.data_validation):
                    validation_samples = self.trainer.nb_val_steps * logs['size']
                elif self.trainer.data_validation:
                    validation_samples = len(self.trainer.data_validation[0])

            # we need to do it in on_batch_begin due to the fact that
            # self.model.validation_data is not availabe in on_train_begin
            self.data_gathered = True
            dataset_infos = {}
            dataset_info = {
                'Training': training_samples,
                'Validation': validation_samples,
            }
            dataset_infos['input1'] = dataset_info
            self.job_backend.set_system_info('datasets', dataset_infos)


        if 'nb_batches' not in self.current:
            batch_size = logs['size']
            if 'samples' in self.params:
                nb_batches = math.ceil(self.params['samples'] / batch_size)  # normal nb batches
            else:
                nb_batches = self.params['steps']<commit_msg>Send parameters count and keras backend as additional info<commit_after># -*- coding: utf-8 -*-
from __future__ import division
from __future__ import absolute_import

import os
import time

from keras.optimizers import Adadelta, Adam, Adamax, Adagrad, RMSprop, SGD

from aetros.backend import JobImage

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO
from io import BytesIO

import PIL.Image
import math
from keras.callbacks import Callback
from keras import backend as K
import keras.layers.convolutional

import numpy as np

from aetros.utils.image import get_layer_vis_square, get_layer_vis_square_raw, get_image_tales
from .keras_model_utils import ensure_dir, get_total_params
import six

class KerasLogger(Callback):
    def __init__(self, trainer, job_backend, general_logger, force_insights=False):
        self.params = {}
        super(KerasLogger, self).__init__()
        self.validation_per_batch = []
        self.insight_layer = []
        self.ins = None
        self.insights_sample_path = None
        self.force_insights = force_insights

        self.trainer = trainer
        self.current = {}
        self.log_epoch = True
        self.confusion_matrix = True

        self.job_backend = job_backend
        self.job_model = job_backend.get_job_model()
        self.general_logger = general_logger
        self._test_with_acc = None
        self.last_batch_time = time.time()
        self.start_time = time.time()
        self.made_batches = 0
        self.batches_per_second = 0
        self.stats = []
        self.last_current = None
        self.filepath_best = self.job_model.get_weights_filepath_best()
        self.filepath_latest = self.job_model.get_weights_filepath_latest()

        ensure_dir(os.path.dirname(self.filepath_best))
        self.data_gathered = False

        self.accuracy_channel = None
        self.all_losses = None
        self.loss_channel = None
        self.learning_rate_channel = None

        self.learning_rate_start = 0

        self.insights_x = None
        self.best_total_accuracy = 0

    def on_train_begin(self, logs={}):
        self.start_time = time.time()
        self.last_batch_time = time.time()
        self.job_backend.set_status('TRAINING')
        self.job_backend.set_info('parameters', get_total_params(self.model))
        self.job_backend.set_info('backend', K.backend())

        if self.model.optimizer and hasattr(self.model.optimizer, 'get_config'):
            config = self.model.optimizer.get_config()
            self.job_backend.set_info('optimizer', type(self.model.optimizer).__name__)
            for i, v in config.items():
                self.job_backend.set_info('optimizer.'+str(i), v)

        self.current['epoch'] = 0
        self.current['started'] = self.start_time
        self.job_backend.set_system_info('current', self.current)

        xaxis = {
            'range': [1, self.params['epochs']],
            'title': u'Epoch ⇢'
        }
        yaxis = {
            'range': [1, 100],
            'title': u'% Accuracy ⇢',
            'dtick': 10
        }

        traces = ['validation', 'training']
        if len(self.model.output_layers) > 1:
            traces = []
            for output in self.model.output_layers:
                traces.append(output.name+'_validation')
                traces.append(output.name+'_training')

        self.accuracy_channel = self.job_backend.create_channel('accuracy', main=True, traces=traces, kpi=True, max_optimization=True, xaxis=xaxis, yaxis=yaxis)
        self.loss_channel = self.job_backend.create_loss_channel('loss', xaxis=xaxis)
        self.learning_rate_channel = self.job_backend.create_channel('learning rate', traces=['start', 'end'], xaxis=xaxis)
        self.job_backend.progress(0, self.params['epochs'])
        if len(self.model.output_layers) > 1:
            loss_traces = []
            for output in self.model.output_layers:
                loss_traces.append(output.name+'_validation')
                loss_traces.append(output.name+'_training')

            self.all_losses = self.job_backend.create_channel('All loss', main=True, xaxis=xaxis, traces=loss_traces)

        images = self.build_insight_images()
        self.job_backend.job_add_insight(0, images, None)

    def on_batch_begin(self, batch, logs={}):

        if not self.data_gathered:
            validation_samples = self.trainer.nb_val_samples
            if 'samples' in self.params:
                training_samples = self.params['samples']
                if hasattr(self.model, 'validation_data'):
                    validation_samples = len(self.model.validation_data[0])
            else:
                training_samples = self.params['steps'] * logs['size']
                if self.trainer.is_generator(self.trainer.data_validation):
                    validation_samples = self.trainer.nb_val_steps * logs['size']
                elif self.trainer.data_validation:
                    validation_samples = len(self.trainer.data_validation[0])

            # we need to do it in on_batch_begin due to the fact that
            # self.model.validation_data is not availabe in on_train_begin
            self.data_gathered = True
            dataset_infos = {}
            dataset_info = {
                'Training': training_samples,
                'Validation': validation_samples,
            }
            dataset_infos['input1'] = dataset_info
            self.job_backend.set_system_info('datasets', dataset_infos)


        if 'nb_batches' not in self.current:
            batch_size = logs['size']
            if 'samples' in self.params:
                nb_batches = math.ceil(self.params['samples'] / batch_size)  # normal nb batches
            else:
                nb_batches = self.params['steps']
{'fraction_non_alphanumeric': 0.09812271858160959, 'fraction_numerical': 0.0035633582478706762, 'mean_word_length': 2.472238986119493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6499549', 'n_tokens_mistral': 3477, 'n_tokens_neox': 3368, 'n_words': 664}