<commit_before>import argparse
import datetime
import os
from time import time

import numpy as np
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator

from TrainingHistoryPlotter import TrainingHistoryPlotter
from datasets.DatasetSplitter import DatasetSplitter
from models.ConfigurationFactory import ConfigurationFactory


def train_model(dataset_directory: str,
                model_name: str,
                show_plot_after_training: bool,
                delete_and_recreate_dataset_directory: bool):

    if delete_and_recreate_dataset_directory:
        dataset_splitter = DatasetSplitter(dataset_directory, dataset_directory)
        dataset_splitter.delete_split_directories()
        dataset_splitter.split_images_into_training_validation_and_test_set()

    print("Training on dataset...")
    start_time = time()

    training_configuration = ConfigurationFactory.get_configuration_by_name(model_name)
    img_rows, img_cols = training_configuration.data_shape[1], training_configuration.data_shape[0]
    number_of_pixels_shift = training_configuration.number_of_pixel_shift

    train_generator = ImageDataGenerator(rotation_range=10,
                                         width_shift_range=number_of_pixels_shift / img_rows,
                                         height_shift_range=number_of_pixels_shift / img_cols,
                                         )
    training_data_generator = train_generator.flow_from_directory(os.path.join(dataset_directory, "training"),
                                                                  target_size=(img_cols, img_rows),
                                                                  batch_size=training_configuration.training_minibatch_size,
                                                                  )
    training_steps_per_epoch = np.math.ceil(training_data_generator.samples / training_data_generator.batch_size)

    validation_generator = ImageDataGenerator()
    validation_data_generator = validation_generator.flow_from_directory(os.path.join(dataset_directory, "validation"),
                                                                         target_size=(img_cols, img_rows),
                                                                         batch_size=training_configuration.training_minibatch_size)
    validation_steps_per_epoch = np.math.ceil(validation_data_generator.samples / validation_data_generator.batch_size)

    test_generator = ImageDataGenerator()
    test_data_generator = test_generator.flow_from_directory(os.path.join(dataset_directory, "test"),
                                                             target_size=(img_cols, img_rows),
                                                             batch_size=training_configuration.training_minibatch_size)
    test_steps_per_epoch = np.math.ceil(test_data_generator.samples / test_data_generator.batch_size)

    model = training_configuration.classifier()
    model.summary()

    print("Model {0} loaded.".format(training_configuration.name()))
    print(training_configuration.summary())

    best_model_path = "{0}.h5".format(training_configuration.name())

    model_checkpoint = ModelCheckpoint(best_model_path, monitor="val_acc", save_best_only=True, verbose=1)
    early_stop = EarlyStopping(monitor='val_acc',
                               patience=training_configuration.number_of_epochs_before_early_stopping,
                               verbose=1)
    learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
                                                patience=training_configuration.number_of_epochs_before_reducing_learning_rate,
                                                verbose=1,
                                                factor=training_configuration.learning_rate_reduction_factor,
                                                min_lr=training_configuration.minimum_learning_rate)
    history = model.fit_generator(
            generator=training_data_generator,
            steps_per_epoch=training_steps_per_epoch,
            epochs=training_configuration.number_of_epochs,
            callbacks=[model_checkpoint, early_stop, learning_rate_reduction],
            validation_data=validation_data_generator,
            validation_steps=validation_steps_per_epoch
    )

    print("Loading best model from check-point and testing...")
    # For some models, loading the model directly does not work, but loading the weights does
    # (see https://github.com/fchollet/keras/issues/4044#issuecomment-254921595)
    # best_model = keras.models.load_model(best_model_path)
    best_model = training_configuration.classifier()
    best_model.load_weights(best_model_path)

    evaluation = best_model.evaluate_generator(test_data_generator, steps=test_steps_per_epoch)

    print(best_model.metrics_names)
    print("Loss : ", evaluation[0])
    print("Accuracy : ", evaluation[1])
    print("Error : ", 1 - evaluation[1])

    end_time = time()<commit_msg>Calculating advanced metrics now for testing phase<commit_after>import argparse
import datetime
import os
from sklearn import metrics
from time import time

import numpy
import numpy as np
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator

from TrainingHistoryPlotter import TrainingHistoryPlotter
from datasets.DatasetSplitter import DatasetSplitter
from models.ConfigurationFactory import ConfigurationFactory


def train_model(dataset_directory: str,
                model_name: str,
                show_plot_after_training: bool,
                delete_and_recreate_dataset_directory: bool):

    if delete_and_recreate_dataset_directory:
        dataset_splitter = DatasetSplitter(dataset_directory, dataset_directory)
        dataset_splitter.delete_split_directories()
        dataset_splitter.split_images_into_training_validation_and_test_set()

    print("Training on dataset...")
    start_time = time()

    training_configuration = ConfigurationFactory.get_configuration_by_name(model_name)
    img_rows, img_cols = training_configuration.data_shape[1], training_configuration.data_shape[0]
    number_of_pixels_shift = training_configuration.number_of_pixel_shift

    train_generator = ImageDataGenerator(rotation_range=10,
                                         width_shift_range=number_of_pixels_shift / img_rows,
                                         height_shift_range=number_of_pixels_shift / img_cols,
                                         )
    training_data_generator = train_generator.flow_from_directory(os.path.join(dataset_directory, "training"),
                                                                  target_size=(img_cols, img_rows),
                                                                  batch_size=training_configuration.training_minibatch_size,
                                                                  )
    training_steps_per_epoch = np.math.ceil(training_data_generator.samples / training_data_generator.batch_size)

    validation_generator = ImageDataGenerator()
    validation_data_generator = validation_generator.flow_from_directory(os.path.join(dataset_directory, "validation"),
                                                                         target_size=(img_cols, img_rows),
                                                                         batch_size=training_configuration.training_minibatch_size)
    validation_steps_per_epoch = np.math.ceil(validation_data_generator.samples / validation_data_generator.batch_size)

    test_generator = ImageDataGenerator()
    test_data_generator = test_generator.flow_from_directory(os.path.join(dataset_directory, "test"),
                                                             target_size=(img_cols, img_rows),
                                                             batch_size=training_configuration.training_minibatch_size,
                                                             shuffle=False)
    test_steps_per_epoch = np.math.ceil(test_data_generator.samples / test_data_generator.batch_size)

    model = training_configuration.classifier()
    model.summary()

    print("Model {0} loaded.".format(training_configuration.name()))
    print(training_configuration.summary())

    best_model_path = "{0}.h5".format(training_configuration.name())

    model_checkpoint = ModelCheckpoint(best_model_path, monitor="val_acc", save_best_only=True, verbose=1)
    early_stop = EarlyStopping(monitor='val_acc',
                               patience=training_configuration.number_of_epochs_before_early_stopping,
                               verbose=1)
    learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
                                                patience=training_configuration.number_of_epochs_before_reducing_learning_rate,
                                                verbose=1,
                                                factor=training_configuration.learning_rate_reduction_factor,
                                                min_lr=training_configuration.minimum_learning_rate)
    history = model.fit_generator(
            generator=training_data_generator,
            steps_per_epoch=training_steps_per_epoch,
            epochs=training_configuration.number_of_epochs,
            callbacks=[model_checkpoint, early_stop, learning_rate_reduction],
            validation_data=validation_data_generator,
            validation_steps=validation_steps_per_epoch
    )

    print("Loading best model from check-point and testing...")
    # For some models, loading the model directly does not work, but loading the weights does
    # (see https://github.com/fchollet/keras/issues/4044#issuecomment-254921595)
    # best_model = keras.models.load_model(best_model_path)
    best_model = training_configuration.classifier()
    best_model.load_weights(best_model_path)

    test_data_generator.reset()
    true_classes = test_data_generator.classes
    predictions = best_model.predict_generator(test_data_generator, steps=test_steps_per_epoch)
    predicted_classes = numpy.argmax(predictions, axis=1)

    report = metrics.classification_report(true_classes, predicted_classes)
    # accuracy = metrics.accuracy_score(true_classes, predicted_classes) # is the same as from evaluate_generator

    test_data_generator.reset()
    evaluation = best_model.evaluate_generator(test_data_generator, steps=test_steps_per_epoch)

    print(report)
    print("Total Loss: {0:.5f}".format(evaluation[0]))
    print("Total Accuracy: {0:0.5f}%".format(evaluation[1] * 100))
    print("Total Error: {0:0.5f}%".format((1 - evaluation[1]) * 100))

    end_time = time()
{'fraction_non_alphanumeric': 0.06540240518038853, 'fraction_numerical': 0.006382978723404255, 'mean_word_length': 2.550410509031199, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 60, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1435050', 'n_tokens_mistral': 2683, 'n_tokens_neox': 2613, 'n_words': 395}