<commit_before>import argparse
import datetime
import fnmatch
import os
import pickle
import re
from time import time
from typing import List

import tensorflow.keras
import numpy
import numpy as np
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from ClassWeightCalculator import ClassWeightCalculator
from reporting import TelegramNotifier, GoogleSpreadsheetReporter, sklearn_reporting
from reporting.TrainingHistoryPlotter import TrainingHistoryPlotter
from datasets.TrainingDatasetProvider import TrainingDatasetProvider
from datasets.DirectoryIteratorWithBoundingBoxes import DirectoryIteratorWithBoundingBoxes
from models.ConfigurationFactory import ConfigurationFactory


def train_model(dataset_directory: str, model_name: str, stroke_thicknesses: List[int],
                width: int, height: int,
                staff_line_vertical_offsets: List[int], training_minibatch_size: int,
                optimizer: str, dynamic_learning_rate_reduction: bool, use_fixed_canvas: bool, datasets: List[str],
                class_weights_balancing_method: str,
                save_after_every_epoch: bool,
                resume_from_checkpoint: str,
                send_telegram_messages: bool):
    image_dataset_directory = os.path.join(dataset_directory, "images")

    bounding_boxes = None
    bounding_boxes_cache = os.path.join(dataset_directory, "bounding_boxes.txt")

    print("Loading configuration and data-readers...")
    start_time = time()

    number_of_classes = len(os.listdir(os.path.join(image_dataset_directory, "training")))
    training_configuration = ConfigurationFactory.get_configuration_by_name(model_name, optimizer, width, height,
                                                                            training_minibatch_size, number_of_classes)
    if training_configuration.performs_localization() and bounding_boxes is None:
        # Try to unpickle
        with open(bounding_boxes_cache, "rb") as cache:
            bounding_boxes = pickle.load(cache)

    if not training_configuration.performs_localization():
        bounding_boxes = None

    train_generator = ImageDataGenerator(rotation_range=training_configuration.rotation_range,
                                         zoom_range=training_configuration.zoom_range
                                         )
    training_data_generator = DirectoryIteratorWithBoundingBoxes(
        directory=os.path.join(image_dataset_directory, "training"),
        image_data_generator=train_generator,
        target_size=(training_configuration.input_image_rows,
                     training_configuration.input_image_columns),
        batch_size=training_configuration.training_minibatch_size,
        bounding_boxes=bounding_boxes,
    )
    training_steps_per_epoch = np.math.ceil(training_data_generator.samples / training_data_generator.batch_size)

    validation_generator = ImageDataGenerator()
    validation_data_generator = DirectoryIteratorWithBoundingBoxes(
        directory=os.path.join(image_dataset_directory, "validation"),
        image_data_generator=validation_generator,
        target_size=(
            training_configuration.input_image_rows,
            training_configuration.input_image_columns),
        batch_size=training_configuration.training_minibatch_size,
        bounding_boxes=bounding_boxes)
    validation_steps_per_epoch = np.math.ceil(validation_data_generator.samples / validation_data_generator.batch_size)

    test_generator = ImageDataGenerator()
    test_data_generator = DirectoryIteratorWithBoundingBoxes(
        directory=os.path.join(image_dataset_directory, "test"),
        image_data_generator=test_generator,
        target_size=(training_configuration.input_image_rows,
                     training_configuration.input_image_columns),
        batch_size=training_configuration.training_minibatch_size,
        shuffle=False,
        bounding_boxes=bounding_boxes)
    test_steps_per_epoch = np.math.ceil(test_data_generator.samples / test_data_generator.batch_size)

    model = training_configuration.classifier()
    model.summary()

    print("Model {0} created.".format(training_configuration.name()))

    initial_epoch = 0
    if resume_from_checkpoint:
        # Try to parse epoch from checkpoint filename. Checkpoint files written by this program
        # are of the form <start>_<configname>-<epoch>.h5.
        # The regular expression assumes there are no dashes in start or configname, otherwise
        # the initial epoch will just be 0. That is harmless unless you have parameters that
        # depend on the epoch.
        m = re.match('[^-]+-(\d+).h5', resume_from_checkpoint)
        if m and m.groups() and len(m.groups() == 1):
            initial_epoch = int(m.groups()[0]) + 1
        # This will not restore parameters that are adapted dynamically during training since
        # afaik not all of them get saved to the model checkpoint.
        model.load_weights(resume_from_checkpoint)
        print("Model {0} weights loaded from checkpoint {1}. Training will resume from epoch {2}".format(
            training_configuration.name(),
            resume_from_checkpoint,
            initial_epoch))

    print(training_configuration.summary())

    start_of_training = datetime.date.today()

    monitor_variable = 'val_accuracy'
    if training_configuration.performs_localization():
        monitor_variable = 'val_output_class_accuracy'

    best_model_path = "{0}_{1}".format(start_of_training, training_configuration.name())
    if save_after_every_epoch:<commit_msg>correct regexp for save file name matching<commit_after>import argparse
import datetime
import fnmatch
import os
import pickle
import re
from time import time
from typing import List

import tensorflow.keras
import numpy
import numpy as np
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from ClassWeightCalculator import ClassWeightCalculator
from reporting import TelegramNotifier, GoogleSpreadsheetReporter, sklearn_reporting
from reporting.TrainingHistoryPlotter import TrainingHistoryPlotter
from datasets.TrainingDatasetProvider import TrainingDatasetProvider
from datasets.DirectoryIteratorWithBoundingBoxes import DirectoryIteratorWithBoundingBoxes
from models.ConfigurationFactory import ConfigurationFactory


def train_model(dataset_directory: str, model_name: str, stroke_thicknesses: List[int],
                width: int, height: int,
                staff_line_vertical_offsets: List[int], training_minibatch_size: int,
                optimizer: str, dynamic_learning_rate_reduction: bool, use_fixed_canvas: bool, datasets: List[str],
                class_weights_balancing_method: str,
                save_after_every_epoch: bool,
                resume_from_checkpoint: str,
                send_telegram_messages: bool):
    image_dataset_directory = os.path.join(dataset_directory, "images")

    bounding_boxes = None
    bounding_boxes_cache = os.path.join(dataset_directory, "bounding_boxes.txt")

    print("Loading configuration and data-readers...")
    start_time = time()

    number_of_classes = len(os.listdir(os.path.join(image_dataset_directory, "training")))
    training_configuration = ConfigurationFactory.get_configuration_by_name(model_name, optimizer, width, height,
                                                                            training_minibatch_size, number_of_classes)
    if training_configuration.performs_localization() and bounding_boxes is None:
        # Try to unpickle
        with open(bounding_boxes_cache, "rb") as cache:
            bounding_boxes = pickle.load(cache)

    if not training_configuration.performs_localization():
        bounding_boxes = None

    train_generator = ImageDataGenerator(rotation_range=training_configuration.rotation_range,
                                         zoom_range=training_configuration.zoom_range
                                         )
    training_data_generator = DirectoryIteratorWithBoundingBoxes(
        directory=os.path.join(image_dataset_directory, "training"),
        image_data_generator=train_generator,
        target_size=(training_configuration.input_image_rows,
                     training_configuration.input_image_columns),
        batch_size=training_configuration.training_minibatch_size,
        bounding_boxes=bounding_boxes,
    )
    training_steps_per_epoch = np.math.ceil(training_data_generator.samples / training_data_generator.batch_size)

    validation_generator = ImageDataGenerator()
    validation_data_generator = DirectoryIteratorWithBoundingBoxes(
        directory=os.path.join(image_dataset_directory, "validation"),
        image_data_generator=validation_generator,
        target_size=(
            training_configuration.input_image_rows,
            training_configuration.input_image_columns),
        batch_size=training_configuration.training_minibatch_size,
        bounding_boxes=bounding_boxes)
    validation_steps_per_epoch = np.math.ceil(validation_data_generator.samples / validation_data_generator.batch_size)

    test_generator = ImageDataGenerator()
    test_data_generator = DirectoryIteratorWithBoundingBoxes(
        directory=os.path.join(image_dataset_directory, "test"),
        image_data_generator=test_generator,
        target_size=(training_configuration.input_image_rows,
                     training_configuration.input_image_columns),
        batch_size=training_configuration.training_minibatch_size,
        shuffle=False,
        bounding_boxes=bounding_boxes)
    test_steps_per_epoch = np.math.ceil(test_data_generator.samples / test_data_generator.batch_size)

    model = training_configuration.classifier()
    model.summary()

    print("Model {0} created.".format(training_configuration.name()))

    initial_epoch = 0
    if resume_from_checkpoint:
        # Try to parse epoch from checkpoint filename. Checkpoint files written by this program
        # are of the form <start>_<configname>-<epoch>.h5.
        # The regular expression assumes there are no dashes in configname, otherwise
        # the initial epoch will just be 0. That is harmless unless you have parameters that
        # depend on the epoch.
        m = re.match('[\d-]+_[^-]+-(\d+).h5', resume_from_checkpoint)
        if m and m.groups() and len(m.groups() == 1):
            initial_epoch = int(m.groups()[0]) + 1
        # This will not restore parameters that are adapted dynamically during training since
        # afaik not all of them get saved to the model checkpoint.
        model.load_weights(resume_from_checkpoint)
        print("Model {0} weights loaded from checkpoint {1}. Training will resume from epoch {2}".format(
            training_configuration.name(),
            resume_from_checkpoint,
            initial_epoch))

    print(training_configuration.summary())

    start_of_training = datetime.date.today()

    monitor_variable = 'val_accuracy'
    if training_configuration.performs_localization():
        monitor_variable = 'val_output_class_accuracy'

    best_model_path = "{0}_{1}".format(start_of_training, training_configuration.name())
    if save_after_every_epoch:
{'fraction_non_alphanumeric': 0.06739302346015288, 'fraction_numerical': 0.0022845092698356912, 'mean_word_length': 3.6438188494492043, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1651371', 'n_tokens_mistral': 2998, 'n_tokens_neox': 2926, 'n_words': 629}