        kwargs_handlers=[DistributedDataParallelKwargs(find_unused_parameters=False),
                         InitProcessGroupKwargs(timeout=timeout)],
    )
    device = accelerator.device
    exp_save_dir = os.path.join(save_dir, get_exp_name(config))

    tokenizer = AutoTokenizer.from_pretrained(base_model, cache_dir=path_to_cache)
    model = AutoModelForCausalLM.from_pretrained(
        base_model, attn_implementation="sdpa", torch_dtype=torch.bfloat16, cache_dir=path_to_cache
    )
    model.gradient_checkpointing_enable()
