<commit_before>"""
This is a training script for sequence to sequence learning.
"""
# tests: lint, mypy

import sys
import os
from shutil import copyfile

import tensorflow as tf

from neuralmonkey.checking import check_dataset_and_coders
from neuralmonkey.logging import Logging, log
from neuralmonkey.config.configuration import Configuration
from neuralmonkey.learning_utils import training_loop, initialize_tf
from neuralmonkey.dataset import Dataset

def create_config(config_file):
    config = Configuration()
    config.add_argument('name', str)
    config.add_argument('random_seed', int, required=False)
    config.add_argument('output', str)
    config.add_argument('epochs', int, cond=lambda x: x >= 0)
    config.add_argument('trainer')
    config.add_argument('encoders', list)
    config.add_argument('decoder')
    config.add_argument('batch_size', int, cond=lambda x: x > 0)
    config.add_argument('train_dataset', Dataset)
    config.add_argument('val_dataset', Dataset)
    config.add_argument('postprocess')
    config.add_argument('evaluation', cond=list)
    config.add_argument('runner')
    config.add_argument('test_datasets', list, required=False, default=[])
    config.add_argument('initial_variables', str, required=False, default=[])
    config.add_argument('validation_period', int, required=False, default=500)
    config.add_argument('logging_period', int, required=False, default=20)
    config.add_argument('threads', int, required=False, default=4)
    config.add_argument('minimize', bool, required=False, default=False)
    config.add_argument('save_n_best', int, required=False, default=1)
    config.add_argument('overwrite_output_dir', bool, required=False,
                        default=False)

    return config.load_file(config_file)

def main():
    if len(sys.argv) != 2:
        print("Usage: train.py <ini_file>")
        exit(1)

    args = create_config(sys.argv[1])

    print("")

    #pylint: disable=no-member,broad-except
    if args.random_seed is not None:
        tf.set_random_seed(args.random_seed)

    if os.path.isdir(args.output) and \
            os.path.exists(os.path.join(args.output, "experiment.ini")):
        if args.overwrite_output_dir:
            # we do not want to delete the directory contents
            log("Directory with experiment.ini '{}' exists, "
                "overwriting enabled, proceeding."
                .format(args.output))
        else:
            log("Directory with experiment.ini '{}' exists, "
                "overwriting disabled."
                .format(args.output), color='red')
            exit(1)

    try:
        check_dataset_and_coders(args.train_dataset,
                                 args.encoders + [args.decoder])
        check_dataset_and_coders(args.val_dataset,
                                 args.encoders + [args.decoder])<commit_msg>hardcode randseed, in a way that actually works<commit_after>"""
This is a training script for sequence to sequence learning.
"""
# tests: lint, mypy

import sys
import os
from shutil import copyfile

import numpy as np
import tensorflow as tf

from neuralmonkey.checking import check_dataset_and_coders
from neuralmonkey.logging import Logging, log
from neuralmonkey.config.configuration import Configuration
from neuralmonkey.learning_utils import training_loop, initialize_tf
from neuralmonkey.dataset import Dataset

def create_config(config_file):
    config = Configuration()
    config.add_argument('name', str)
    config.add_argument('random_seed', int, required=False)
    config.add_argument('output', str)
    config.add_argument('epochs', int, cond=lambda x: x >= 0)
    config.add_argument('trainer')
    config.add_argument('encoders', list)
    config.add_argument('decoder')
    config.add_argument('batch_size', int, cond=lambda x: x > 0)
    config.add_argument('train_dataset', Dataset)
    config.add_argument('val_dataset', Dataset)
    config.add_argument('postprocess')
    config.add_argument('evaluation', cond=list)
    config.add_argument('runner')
    config.add_argument('test_datasets', list, required=False, default=[])
    config.add_argument('initial_variables', str, required=False, default=[])
    config.add_argument('validation_period', int, required=False, default=500)
    config.add_argument('logging_period', int, required=False, default=20)
    config.add_argument('threads', int, required=False, default=4)
    config.add_argument('minimize', bool, required=False, default=False)
    config.add_argument('save_n_best', int, required=False, default=1)
    config.add_argument('overwrite_output_dir', bool, required=False,
                        default=False)

    return config.load_file(config_file)

def main():
    if len(sys.argv) != 2:
        print("Usage: train.py <ini_file>")
        exit(1)

    # random seeds have to be set before anything is created in the graph
    np.random.seed(1)
    tf.set_random_seed(1)

    args = create_config(sys.argv[1])

    print("")

    #pylint: disable=no-member,broad-except
    if args.random_seed is not None:
        tf.set_random_seed(args.random_seed)

    if os.path.isdir(args.output) and \
            os.path.exists(os.path.join(args.output, "experiment.ini")):
        if args.overwrite_output_dir:
            # we do not want to delete the directory contents
            log("Directory with experiment.ini '{}' exists, "
                "overwriting enabled, proceeding."
                .format(args.output))
        else:
            log("Directory with experiment.ini '{}' exists, "
                "overwriting disabled."
                .format(args.output), color='red')
            exit(1)

    try:
        check_dataset_and_coders(args.train_dataset,
                                 args.encoders + [args.decoder])
        check_dataset_and_coders(args.val_dataset,
                                 args.encoders + [args.decoder])
{'fraction_non_alphanumeric': 0.11143050388907677, 'fraction_numerical': 0.0047345282380791345, 'mean_word_length': 3.3685376661742983, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6679715', 'n_tokens_mistral': 1751, 'n_tokens_neox': 1709, 'n_words': 381}