@utils.timeit
def run_trainer(model_dir, algorithm_name, arch, clustering_file, distributed, dist_protocol):
    model_path = Path(model_dir)
    model_path.mkdir(parents=True, exist_ok=True)
    train_cfg = OmegaConf.create(TRAIN_CFG)
    OmegaConf.set_struct(train_cfg, True)
    train_cfg.output_dir = model_dir
    train_cfg.algorithm.name = algorithm_name
    train_cfg.algorithm.arch = arch
    train_cfg.experiment.distributed = distributed
    train_cfg.experiment.dist_protocol = dist_protocol
    with open((model_path / 'train_cfg.yaml'), 'w') as fp:
        OmegaConf.save(train_cfg, fp.name)
    with filelock.FileLock((clustering_file + '.lock')):
        with open(clustering_file, 'rb') as fp:
            clustering = pickle.load(fp)
    datasets = partition_datasets(train_cfg, partitions=clustering['partitions'])
    Algorithm = utils.load_model_cls(train_cfg)
    trainer_args = (train_cfg, Algorithm, datasets['train']['in'])
    output = train_algorithm(*trainer_args)
    return utils.pack(output)
@utils.timeit
def run_trainer(model_dir, algorithm_name, arch, clustering_file, distributed, dist_protocol):
    model_path = Path(model_dir)
    model_path.mkdir(parents=True, exist_ok=True)
    train_cfg = OmegaConf.create(TRAIN_CFG)
    OmegaConf.set_struct(train_cfg, True)
    train_cfg.output_dir = model_dir
    train_cfg.algorithm.name = algorithm_name
    train_cfg.algorithm.arch = arch
    train_cfg.experiment.distributed = distributed
    train_cfg.experiment.dist_protocol = dist_protocol
    with open((model_path / 'train_cfg.yaml'), 'w') as fp:
        OmegaConf.save(train_cfg, fp.name)
    with filelock.FileLock((clustering_file + '.lock')):
        with open(clustering_file, 'rb') as fp:
            clustering = pickle.load(fp)
    datasets = partition_datasets(train_cfg, partitions=clustering['partitions'])
    Algorithm = utils.load_model_cls(train_cfg)
    trainer_args = (train_cfg, Algorithm, datasets['train']['in'])
    output = train_algorithm(*trainer_args)
    return utils.pack(output)