def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('config', help='Configuration yaml file')
    parser.add_argument('-p', '--proc', type=int, default=1, help='Number of processors')
    args = parser.parse_args()
    if (not exists(args.config)):
        raise FileNotFoundError((args.config + ' not found.'))
    with open(args.config) as config_file:
        config = yaml.load(config_file)
    (train_files, val_files, test_files) = subset_data_files_by_date(config['data_path'], config['data_end'], **config['subset_data'])
    input_scaler = scalers[config['input_scaler']]()
    (train_input, train_output_labels, train_transformed_output, train_scaled_output, output_scalers) = assemble_data_files(train_files, config['input_cols'], config['output_cols'], config['input_transforms'], config['output_transforms'], input_scaler, subsample=config['subsample'])
    print('Train Input Size:', train_input.shape)
    (val_input, val_output_labels, val_transformed_output, val_scaled_output, output_scalers) = assemble_data_files(val_files, config['input_cols'], config['output_cols'], config['input_transforms'], config['output_transforms'], input_scaler, output_scalers=output_scalers, train=False, subsample=config['subsample'])
    print('Val Input Size:', val_input.shape)
    cluster = LocalCluster(n_workers=args.proc, threads_per_worker=1)
    client = Client(cluster)
    print(client)
    train_input_link = client.scatter(train_input)
    train_labels_link = client.scatter(train_output_labels)
    train_scaled_output_link = client.scatter(train_scaled_output)
    val_input_link = client.scatter(val_input)
    val_output_labels_link = client.scatter(val_output_labels)
    val_scaled_output_link = client.scatter(val_scaled_output)
    submissions = []
    if (not exists(config['out_path'])):
        makedirs(config['out_path'])
    for (class_model_name, class_model_params) in config['classifier_models'].items():
        for (reg_model_name, reg_model_params) in config['regressor_models'].items():
            rs = np.random.RandomState(config['random_seed'])
            class_model_config_generator = parse_model_config_params(class_model_params, config['num_param_samples'], rs)
            reg_model_config_generator = parse_model_config_params(reg_model_params, config['num_param_samples'], rs)
            class_model_configs = []
            reg_model_configs = []
            for s in range(config['num_param_samples']):
                class_model_config = next(class_model_config_generator)
                reg_model_config = next(reg_model_config_generator)
                class_model_configs.append(class_model_config)
                reg_model_configs.append(reg_model_config)
                config_index = f'{class_model_name}_{reg_model_name}_{s:04}'
                submissions.append(client.submit(validate_model_configuration, class_model_name, class_model_config, reg_model_name, reg_model_config, config_index, train_input_link, train_labels_link, train_scaled_output_link, val_input_link, val_output_labels_link, val_scaled_output_link, config['classifier_metrics'], config['regressor_metrics']))
            class_config_frame = pd.DataFrame(class_model_configs)
            reg_config_frame = pd.DataFrame(reg_model_configs)
            class_config_frame.to_csv(join(config['out_path'], f'{class_model_name}_{reg_model_name}_classifier_params.csv'), index_label='Config')
            reg_config_frame.to_csv(join(config['out_path'], f'{class_model_name}_{reg_model_name}_regressor_params.csv'))
            result_count = 0
            for out in as_completed(submissions):
                if (out.status == 'finished'):
                    result = out.result()
                    print(result)
                    if (result_count == 0):
                        result.to_frame().T.to_csv(join(config['out_path'], f'{class_model_name}_{reg_model_name}_metrics.csv'), index_label='Config')
                    else:
                        result.to_frame().T.to_csv(join(config['out_path'], f'{class_model_name}_{reg_model_name}_metrics.csv'), header=False, mode='a')
                    result_count += 1
                else:
                    tb = out.traceback()
                    for line in traceback.format_tb(tb):
                        print(line)
            del submissions[:]
    client.close()
    cluster.close()
    return
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('config', help='Configuration yaml file')
    parser.add_argument('-p', '--proc', type=int, default=1, help='Number of processors')
    args = parser.parse_args()
    if (not exists(args.config)):
        raise FileNotFoundError((args.config + ' not found.'))
    with open(args.config) as config_file:
        config = yaml.load(config_file)
    (train_files, val_files, test_files) = subset_data_files_by_date(config['data_path'], config['data_end'], **config['subset_data'])
    input_scaler = scalers[config['input_scaler']]()
    (train_input, train_output_labels, train_transformed_output, train_scaled_output, output_scalers) = assemble_data_files(train_files, config['input_cols'], config['output_cols'], config['input_transforms'], config['output_transforms'], input_scaler, subsample=config['subsample'])
    print('Train Input Size:', train_input.shape)
    (val_input, val_output_labels, val_transformed_output, val_scaled_output, output_scalers) = assemble_data_files(val_files, config['input_cols'], config['output_cols'], config['input_transforms'], config['output_transforms'], input_scaler, output_scalers=output_scalers, train=False, subsample=config['subsample'])
    print('Val Input Size:', val_input.shape)
    cluster = LocalCluster(n_workers=args.proc, threads_per_worker=1)
    client = Client(cluster)
    print(client)
    train_input_link = client.scatter(train_input)
    train_labels_link = client.scatter(train_output_labels)
    train_scaled_output_link = client.scatter(train_scaled_output)
    val_input_link = client.scatter(val_input)
    val_output_labels_link = client.scatter(val_output_labels)
    val_scaled_output_link = client.scatter(val_scaled_output)
    submissions = []
    if (not exists(config['out_path'])):
        makedirs(config['out_path'])
    for (class_model_name, class_model_params) in config['classifier_models'].items():
        for (reg_model_name, reg_model_params) in config['regressor_models'].items():
            rs = np.random.RandomState(config['random_seed'])
            class_model_config_generator = parse_model_config_params(class_model_params, config['num_param_samples'], rs)
            reg_model_config_generator = parse_model_config_params(reg_model_params, config['num_param_samples'], rs)
            class_model_configs = []
            reg_model_configs = []
            for s in range(config['num_param_samples']):
                class_model_config = next(class_model_config_generator)
                reg_model_config = next(reg_model_config_generator)
                class_model_configs.append(class_model_config)
                reg_model_configs.append(reg_model_config)
                config_index = f'{class_model_name}_{reg_model_name}_{s:04}'
                submissions.append(client.submit(validate_model_configuration, class_model_name, class_model_config, reg_model_name, reg_model_config, config_index, train_input_link, train_labels_link, train_scaled_output_link, val_input_link, val_output_labels_link, val_scaled_output_link, config['classifier_metrics'], config['regressor_metrics']))
            class_config_frame = pd.DataFrame(class_model_configs)
            reg_config_frame = pd.DataFrame(reg_model_configs)
            class_config_frame.to_csv(join(config['out_path'], f'{class_model_name}_{reg_model_name}_classifier_params.csv'), index_label='Config')
            reg_config_frame.to_csv(join(config['out_path'], f'{class_model_name}_{reg_model_name}_regressor_params.csv'))
            result_count = 0
            for out in as_completed(submissions):
                if (out.status == 'finished'):
                    result = out.result()
                    print(result)
                    if (result_count == 0):
                        result.to_frame().T.to_csv(join(config['out_path'], f'{class_model_name}_{reg_model_name}_metrics.csv'), index_label='Config')
                    else:
                        result.to_frame().T.to_csv(join(config['out_path'], f'{class_model_name}_{reg_model_name}_metrics.csv'), header=False, mode='a')
                    result_count += 1
                else:
                    tb = out.traceback()
                    for line in traceback.format_tb(tb):
                        print(line)
            del submissions[:]
    client.close()
    cluster.close()
    return