<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016-2017 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# <NAME> - http://herve.niderb.fr


import matplotlib
import warnings
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    matplotlib.use('Agg')
import matplotlib.pyplot as plt

import os
import pickle
import os.path
import datetime
import numpy as np
from keras.callbacks import Callback
import keras.optimizers
from pyannote.audio.util import mkdir_p


class LoggingCallback(Callback):
    """Logging callback

    Parameters
    ----------
    log_dir : str
    log : list of tuples, optional
        Defaults to [('train', 'loss')]
    extract_embedding : func
        Function that takes Keras model as input and returns the actual model.
        This is useful for embedding that are not directly optimized by Keras.
        Defaults to identity function.
    """

    ARCHITECTURE_YML = '{log_dir}/architecture.yml'
    WEIGHTS_DIR = '{log_dir}/weights'
    WEIGHTS_H5 = '{log_dir}/weights/{epoch:04d}.h5'
    OPTIMIZER_DIR = '{log_dir}/optimizer'
    OPTIMIZER_PKL = '{log_dir}/optimizer/{epoch:04d}.pkl'
    LOG_TXT = '{log_dir}/{name}.{subset}.txt'
    LOG_PNG = '{log_dir}/{name}.{subset}.png'
    LOG_EPS = '{log_dir}/{name}.{subset}.eps'

    def __init__(self, log_dir, log=None, extract_embedding=None):
        super(LoggingCallback, self).__init__()

        # make sure path is absolute
        self.log_dir = os.path.realpath(log_dir)

        if extract_embedding is None:
            extract_embedding = lambda model: model
        self.extract_embedding = extract_embedding

        # create log_dir directory (and subdirectory)
        try:
            os.makedirs(self.log_dir)
        except OSError as e:
            # this happens when log_dir already exists.
            # we need this **not** to fail because this directory
            # may contain pre-computed (cached) sequences
            pass

        # this will fail if the directory already exists
        # and this is OK  because 'weights' directory
        # usually contains the output of very long computations
        # and you do not want to erase them by mistake :/
        weights_dir = self.WEIGHTS_DIR.format(log_dir=self.log_dir)
        os.makedirs(weights_dir)

        optimizer_dir = self.OPTIMIZER_DIR.format(log_dir=self.log_dir)
        mkdir_p(optimizer_dir)

        if log is None:
            log = [('train', 'loss')]
        self.log = log

        self.values = {}
        for subset, name in self.log:
            if subset not in self.values:
                self.values[subset] = {}
            if name not in self.values[subset]:
                self.values[subset][name] = []

    def get_loss(self, epoch, subset, logs={}):
        if subset != 'train':
            raise ValueError('"loss" is only available for "train"')
        value = logs['loss']
        minimize = True
        return value, minimize

    def get_accuracy(self, epoch, subset, logs={}):
        if subset != 'train':
            raise ValueError('"accuracy" is only available for "train"')
        value = logs['acc']
        minimize = False
        return value, minimize

    def get_value(self, epoch, name, subset, logs={}):
        get_value = getattr(self, 'get_' + name)
        value, minimize = get_value(epoch, subset, logs=logs)
        return value, minimize

    def on_epoch_begin(self, epoch, logs={}):
        """Save architecture before first epoch"""

        if epoch > 0:
            return

        architecture_yml = self.ARCHITECTURE_YML.format(log_dir=self.log_dir)
        model = self.extract_embedding(self.model)
        yaml_string = model.to_yaml()
        with open(architecture_yml, 'w') as fp:
            fp.write(yaml_string)

    def on_epoch_end(self, epoch, logs={}):
        """Save weights (and various curves) after each epoch"""

        # keep track of when the epoch ended
        now = datetime.datetime.now().isoformat()

        # save model after this epoch
        weights_h5 = self.WEIGHTS_H5.format(log_dir=self.log_dir, epoch=epoch)

        # save current weights
        try:
            model = self.extract_embedding(self.model)
            model.save_weights(weights_h5, overwrite=True)
        except Exception as e:
            pass

        # save optimizer state every 10 epochs
        if epoch % 10 == 0:
            optimizer = self.model.optimizer
            state = {'optimizer_config': keras.optimizers.serialize(optimizer),
                     'weights': optimizer.get_weights()}

            optimizer_pkl = self.OPTIMIZER_PKL.format(
                log_dir=self.log_dir, epoch=epoch)
            with open(optimizer_pkl, mode='wb') as fp:
                pickle.dump(state, fp)

        for subset, name in self.log:

            value, minimize = self.get_value(epoch, name, subset, logs=logs)

            # keep track of value after last epoch
            self.values[subset][name].append(value)
            values = self.values[subset][name]

            # write value to file
            log_txt = self.LOG_TXT.format(
                log_dir=self.log_dir, name=name, subset=subset)
            TXT_TEMPLATE = '{epoch:d} ' + now + ' {value:.3f}\n'

            mode = 'a' if epoch > 0 else 'w'
            try:
                with open(log_txt, mode) as fp:
                    fp.write(TXT_TEMPLATE.format(epoch=epoch, value=value))
                    fp.flush()
            except Exception as e:
                pass

            # keep track of 'best value' model
            if minimize:
                best_epoch = np.argmin(values)
                best_value = np.min(values)
            else:
                best_epoch = np.argmax(values)
                best_value = np.max(values)

            # plot values to file and mark best value so far
            plt.plot(values, 'b')
            plt.plot([best_epoch], [best_value], 'bo')

            plt.xlabel('epoch')

            YLABEL = '{name} on {subset}'
            ylabel = YLABEL.format(name=name, subset=subset)
            plt.ylabel(ylabel)

            TITLE = '{name} = {best_value:.3f} on {subset} @ epoch #{best_epoch:d}'
            title = TITLE.format(name=name, subset=subset, best_value=best_value, best_epoch=best_epoch)
            plt.title(title)

            plt.tight_layout()

            # save plot as PNG
            log_png = self.LOG_PNG.format(
                log_dir=self.log_dir, name=name, subset=subset)
            try:
                plt.savefig(log_png, dpi=150)
            except Exception as e:
                pass

            # save plot as EPS
            log_eps = self.LOG_EPS.format(
                log_dir=self.log_dir, name=name, subset=subset)
            try:
                plt.savefig(log_eps)
            except Exception as e:
                pass

            plt.close()
<commit_msg>feat: add 'restart' option to LoggingCallback<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016-2017 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# <NAME> - http://herve.niderb.fr


import matplotlib
import warnings
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    matplotlib.use('Agg')
import matplotlib.pyplot as plt

import os
import pickle
import os.path
import datetime
import numpy as np
from keras.callbacks import Callback
import keras.optimizers
from pyannote.audio.util import mkdir_p


class LoggingCallback(Callback):
    """Logging callback

    Parameters
    ----------
    log_dir : str
    log : list of tuples, optional
        Defaults to [('train', 'loss')]
    extract_embedding : func
        Function that takes Keras model as input and returns the actual model.
        This is useful for embedding that are not directly optimized by Keras.
        Defaults to identity function.
    restart : boolean, optional
        Indicates that this training is a restart, not a cold start (default).
    """

    ARCHITECTURE_YML = '{log_dir}/architecture.yml'
    WEIGHTS_DIR = '{log_dir}/weights'
    WEIGHTS_H5 = '{log_dir}/weights/{epoch:04d}.h5'
    OPTIMIZER_DIR = '{log_dir}/optimizer'
    OPTIMIZER_PKL = '{log_dir}/optimizer/{epoch:04d}.pkl'
    LOG_TXT = '{log_dir}/{name}.{subset}.txt'
    LOG_PNG = '{log_dir}/{name}.{subset}.png'
    LOG_EPS = '{log_dir}/{name}.{subset}.eps'

    def __init__(self, log_dir, log=None, extract_embedding=None,
                 restart=False):
        super(LoggingCallback, self).__init__()

        # make sure path is absolute
        self.log_dir = os.path.realpath(log_dir)

        if extract_embedding is None:
            extract_embedding = lambda model: model
        self.extract_embedding = extract_embedding

        # create log_dir directory (and subdirectory)
        try:
            os.makedirs(self.log_dir)
        except OSError as e:
            # this happens when log_dir already exists.
            # we need this **not** to fail because this directory
            # may contain pre-computed (cached) sequences
            pass

        # this will fail if the directory already exists
        # and this is OK  because 'weights' directory
        # usually contains the output of very long computations
        # and you do not want to erase them by mistake :/
        if not restart:
            weights_dir = self.WEIGHTS_DIR.format(log_dir=self.log_dir)
            os.makedirs(weights_dir)

        optimizer_dir = self.OPTIMIZER_DIR.format(log_dir=self.log_dir)
        mkdir_p(optimizer_dir)

        if log is None:
            log = [('train', 'loss')]
        self.log = log

        self.values = {}
        for subset, name in self.log:
            if subset not in self.values:
                self.values[subset] = {}
            if name not in self.values[subset]:
                self.values[subset][name] = []

    def get_loss(self, epoch, subset, logs={}):
        if subset != 'train':
            raise ValueError('"loss" is only available for "train"')
        value = logs['loss']
        minimize = True
        return value, minimize

    def get_accuracy(self, epoch, subset, logs={}):
        if subset != 'train':
            raise ValueError('"accuracy" is only available for "train"')
        value = logs['acc']
        minimize = False
        return value, minimize

    def get_value(self, epoch, name, subset, logs={}):
        get_value = getattr(self, 'get_' + name)
        value, minimize = get_value(epoch, subset, logs=logs)
        return value, minimize

    def on_epoch_begin(self, epoch, logs={}):
        """Save architecture before first epoch"""

        if epoch > 0:
            return

        architecture_yml = self.ARCHITECTURE_YML.format(log_dir=self.log_dir)
        model = self.extract_embedding(self.model)
        yaml_string = model.to_yaml()
        with open(architecture_yml, 'w') as fp:
            fp.write(yaml_string)

    def on_epoch_end(self, epoch, logs={}):
        """Save weights (and various curves) after each epoch"""

        # keep track of when the epoch ended
        now = datetime.datetime.now().isoformat()

        # save model after this epoch
        weights_h5 = self.WEIGHTS_H5.format(log_dir=self.log_dir, epoch=epoch)

        # save current weights
        try:
            model = self.extract_embedding(self.model)
            model.save_weights(weights_h5, overwrite=True)
        except Exception as e:
            pass

        # save optimizer state every 10 epochs
        if epoch % 10 == 0:
            optimizer = self.model.optimizer
            state = {'optimizer_config': keras.optimizers.serialize(optimizer),
                     'weights': optimizer.get_weights()}

            optimizer_pkl = self.OPTIMIZER_PKL.format(
                log_dir=self.log_dir, epoch=epoch)
            with open(optimizer_pkl, mode='wb') as fp:
                pickle.dump(state, fp)

        for subset, name in self.log:

            value, minimize = self.get_value(epoch, name, subset, logs=logs)

            # keep track of value after last epoch
            self.values[subset][name].append(value)
            values = self.values[subset][name]

            # write value to file
            log_txt = self.LOG_TXT.format(
                log_dir=self.log_dir, name=name, subset=subset)
            TXT_TEMPLATE = '{epoch:d} ' + now + ' {value:.3f}\n'

            mode = 'a' if epoch > 0 else 'w'
            try:
                with open(log_txt, mode) as fp:
                    fp.write(TXT_TEMPLATE.format(epoch=epoch, value=value))
                    fp.flush()
            except Exception as e:
                pass

            # keep track of 'best value' model
            if minimize:
                best_epoch = np.argmin(values)
                best_value = np.min(values)
            else:
                best_epoch = np.argmax(values)
                best_value = np.max(values)

            # plot values to file and mark best value so far
            plt.plot(values, 'b')
            plt.plot([best_epoch], [best_value], 'bo')

            plt.xlabel('epoch')

            YLABEL = '{name} on {subset}'
            ylabel = YLABEL.format(name=name, subset=subset)
            plt.ylabel(ylabel)

            TITLE = '{name} = {best_value:.3f} on {subset} @ epoch #{best_epoch:d}'
            title = TITLE.format(name=name, subset=subset, best_value=best_value, best_epoch=best_epoch)
            plt.title(title)

            plt.tight_layout()

            # save plot as PNG
            log_png = self.LOG_PNG.format(
                log_dir=self.log_dir, name=name, subset=subset)
            try:
                plt.savefig(log_png, dpi=150)
            except Exception as e:
                pass

            # save plot as EPS
            log_eps = self.LOG_EPS.format(
                log_dir=self.log_dir, name=name, subset=subset)
            try:
                plt.savefig(log_eps)
            except Exception as e:
                pass

            plt.close()

{'fraction_non_alphanumeric': 0.09355856692036384, 'fraction_numerical': 0.0037126415444588825, 'mean_word_length': 2.224660814046289, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3798352', 'n_tokens_mistral': 4743, 'n_tokens_neox': 4556, 'n_words': 1416}