def main():
    parser = argparse.ArgumentParser()
    parser = HfArgumentParser((ModelArguments, MultiLingAdapterArguments))
    (args, adapter_args) = parser.parse_args_into_dataclasses()
    if (os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and (not args.overwrite_output_dir)):
        raise ValueError('Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.'.format(args.output_dir))
    if (args.server_ip and args.server_port):
        import ptvsd
        print('Waiting for debugger attach')
        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
        ptvsd.wait_for_attach()
    if ((args.local_rank == (- 1)) or args.no_cuda):
        device = torch.device(('cuda' if (torch.cuda.is_available() and (not args.no_cuda)) else 'cpu'))
        args.n_gpu = torch.cuda.device_count()
    else:
        torch.cuda.set_device(args.local_rank)
        device = torch.device('cuda', args.local_rank)
        torch.distributed.init_process_group(backend='nccl')
        args.n_gpu = 1
    args.device = device
    logging.basicConfig(handlers=[logging.FileHandler(args.log_file), logging.StreamHandler()], format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=(logging.INFO if (args.local_rank in [(- 1), 0]) else logging.WARN))
    logging.info(('Input args: %r' % args))
    logger.warning('Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s', args.local_rank, device, args.n_gpu, bool((args.local_rank != (- 1))), args.fp16)
    set_seed(args)
    labels = get_labels(args.labels)
    num_labels = len(labels)
    pad_token_label_id = CrossEntropyLoss().ignore_index
    if (args.local_rank not in [(- 1), 0]):
        torch.distributed.barrier()
    args.do_save_full_model = (not adapter_args.train_adapter)
    args.do_save_adapters = adapter_args.train_adapter
    if args.do_save_adapters:
        logging.info('save adapters')
        logging.info(adapter_args.train_adapter)
    if args.do_save_full_model:
        logging.info('save model')
    if (args.local_rank == 0):
        torch.distributed.barrier()
    logger.info('Training/evaluation parameters %s', args)
    if args.do_train:
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        if adapter_args.train_adapter:
            (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model)
            logger.info('lang adapter names: {}'.format(' '.join(lang_adapter_names)))
        else:
            lang_adatper_names = []
            task_name = None
        model.to(args.device)
        train_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode='train', lang=args.train_langs, lang2id=lang2id, few_shot=args.few_shot)
        (global_step, tr_loss) = train(args, train_dataset, model, tokenizer, labels, pad_token_label_id, lang_adapter_names, task_name, lang2id)
        logger.info(' global_step = %s, average loss = %s', global_step, tr_loss)
    if (args.do_train and ((args.local_rank == (- 1)) or (torch.distributed.get_rank() == 0))):
        if ((not os.path.exists(args.output_dir)) and (args.local_rank in [(- 1), 0])):
            os.makedirs(args.output_dir)
        logger.info('Saving model checkpoint to %s', args.output_dir)
        model_to_save = (model.module if hasattr(model, 'module') else model)
        if args.do_save_adapters:
            logging.info('Save adapter')
            model_to_save.save_all_adapters(args.output_dir)
        if args.do_save_adapter_fusions:
            logging.info('Save adapter fusion')
            model_to_save.save_all_adapter_fusions(args.output_dir)
        if args.do_save_full_model:
            logging.info('Save full model')
            model_to_save.save_pretrained(args.output_dir)
        tokenizer.save_pretrained(args.output_dir)
        torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
    results = {}
    if args.init_checkpoint:
        best_checkpoint = args.init_checkpoint
    elif os.path.exists(os.path.join(args.output_dir, 'checkpoint-best')):
        best_checkpoint = os.path.join(args.output_dir, 'checkpoint-best')
    else:
        best_checkpoint = args.output_dir
    if (args.do_eval and (args.local_rank in [(- 1), 0])):
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        logger.info('Evaluating the model on dev set of training language(en)')
        load_adapter = (((best_checkpoint + '/') + args.task_name) if (args.predict_task_adapter is None) else args.predict_task_adapter)
        logger.info(f'Task Adapter will be loaded from this path {load_adapter}')
        model.model_name = args.model_name_or_path
        (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model, load_adapter=load_adapter)
        model.to(args.device)
        (result, _) = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode='dev', prefix='debugging', lang=args.train_langs, lang2id=lang2id, lang_adapter_names=lang_adapter_names, task_name=task_name, calc_weight_step=args.calc_weight_step)
        results.update(result)
        output_eval_file = os.path.join(args.output_dir, 'eval_results.txt')
        with open(output_eval_file, 'w') as writer:
            for key in sorted(results.keys()):
                writer.write('{} = {}\n'.format(key, str(results[key])))
    if (args.do_predict and (args.local_rank in [(- 1), 0])):
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        logger.info('Evaluating the model on test set of all the languages specified')
        if (adapter_args.train_adapter or args.test_adapter):
            load_adapter = (((best_checkpoint + '/') + args.task_name) if (args.predict_task_adapter is None) else args.predict_task_adapter)
            logger.info(f'Task Adapter will be loaded from this path {load_adapter}')
            load_lang_adapter = args.predict_lang_adapter
            model.model_name = args.model_name_or_path
            (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model, load_adapter=load_adapter, load_lang_adapter=load_lang_adapter)
        model.to(args.device)
        predict_and_save(args, adapter_args, model, tokenizer, labels, lang2id, pad_token_label_id, lang_adapter_names, task_name, 'test')
    if (args.do_predict_train and (args.local_rank in [(- 1), 0])):
        logger.info('Evaluating on the train set of all specified languages')
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        if (adapter_args.train_adapter or args.test_adapter):
            load_adapter = (((best_checkpoint + '/') + args.task_name) if (args.predict_task_adapter is None) else args.predict_task_adapter)
            logger.info(f'Task Adapter will be loaded from this path {load_adapter}')
            load_lang_adapter = args.predict_lang_adapter
            model.model_name = args.model_name_or_path
            (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model, load_adapter=load_adapter, load_lang_adapter=load_lang_adapter)
        model.to(args.device)
        predict_and_save(args, adapter_args, model, tokenizer, labels, lang2id, pad_token_label_id, lang_adapter_names, task_name, 'train')
    if (args.do_predict_dev and (args.local_rank in [(- 1), 0])):
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        logger.info('Evaluating on the dev sets of all the specified languages')
        if (adapter_args.train_adapter or args.test_adapter):
            load_adapter = (((best_checkpoint + '/') + args.task_name) if (args.predict_task_adapter is None) else args.predict_task_adapter)
            logger.info(f'Task Adapter will be loaded from this path {load_adapter}')
            load_lang_adapter = args.predict_lang_adapter
            model.model_name = args.model_name_or_path
            (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model, load_adapter=load_adapter, load_lang_adapter=load_lang_adapter)
        model.to(args.device)
        predict_and_save(args, adapter_args, model, tokenizer, labels, lang2id, pad_token_label_id, lang_adapter_names, task_name, 'dev')
def main():
    parser = argparse.ArgumentParser()
    parser = HfArgumentParser((ModelArguments, MultiLingAdapterArguments))
    (args, adapter_args) = parser.parse_args_into_dataclasses()
    if (os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and (not args.overwrite_output_dir)):
        raise ValueError('Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.'.format(args.output_dir))
    if (args.server_ip and args.server_port):
        import ptvsd
        print('Waiting for debugger attach')
        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
        ptvsd.wait_for_attach()
    if ((args.local_rank == (- 1)) or args.no_cuda):
        device = torch.device(('cuda' if (torch.cuda.is_available() and (not args.no_cuda)) else 'cpu'))
        args.n_gpu = torch.cuda.device_count()
    else:
        torch.cuda.set_device(args.local_rank)
        device = torch.device('cuda', args.local_rank)
        torch.distributed.init_process_group(backend='nccl')
        args.n_gpu = 1
    args.device = device
    logging.basicConfig(handlers=[logging.FileHandler(args.log_file), logging.StreamHandler()], format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=(logging.INFO if (args.local_rank in [(- 1), 0]) else logging.WARN))
    logging.info(('Input args: %r' % args))
    logger.warning('Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s', args.local_rank, device, args.n_gpu, bool((args.local_rank != (- 1))), args.fp16)
    set_seed(args)
    labels = get_labels(args.labels)
    num_labels = len(labels)
    pad_token_label_id = CrossEntropyLoss().ignore_index
    if (args.local_rank not in [(- 1), 0]):
        torch.distributed.barrier()
    args.do_save_full_model = (not adapter_args.train_adapter)
    args.do_save_adapters = adapter_args.train_adapter
    if args.do_save_adapters:
        logging.info('save adapters')
        logging.info(adapter_args.train_adapter)
    if args.do_save_full_model:
        logging.info('save model')
    if (args.local_rank == 0):
        torch.distributed.barrier()
    logger.info('Training/evaluation parameters %s', args)
    if args.do_train:
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        if adapter_args.train_adapter:
            (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model)
            logger.info('lang adapter names: {}'.format(' '.join(lang_adapter_names)))
        else:
            lang_adatper_names = []
            task_name = None
        model.to(args.device)
        train_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode='train', lang=args.train_langs, lang2id=lang2id, few_shot=args.few_shot)
        (global_step, tr_loss) = train(args, train_dataset, model, tokenizer, labels, pad_token_label_id, lang_adapter_names, task_name, lang2id)
        logger.info(' global_step = %s, average loss = %s', global_step, tr_loss)
    if (args.do_train and ((args.local_rank == (- 1)) or (torch.distributed.get_rank() == 0))):
        if ((not os.path.exists(args.output_dir)) and (args.local_rank in [(- 1), 0])):
            os.makedirs(args.output_dir)
        logger.info('Saving model checkpoint to %s', args.output_dir)
        model_to_save = (model.module if hasattr(model, 'module') else model)
        if args.do_save_adapters:
            logging.info('Save adapter')
            model_to_save.save_all_adapters(args.output_dir)
        if args.do_save_adapter_fusions:
            logging.info('Save adapter fusion')
            model_to_save.save_all_adapter_fusions(args.output_dir)
        if args.do_save_full_model:
            logging.info('Save full model')
            model_to_save.save_pretrained(args.output_dir)
        tokenizer.save_pretrained(args.output_dir)
        torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
    results = {}
    if args.init_checkpoint:
        best_checkpoint = args.init_checkpoint
    elif os.path.exists(os.path.join(args.output_dir, 'checkpoint-best')):
        best_checkpoint = os.path.join(args.output_dir, 'checkpoint-best')
    else:
        best_checkpoint = args.output_dir
    if (args.do_eval and (args.local_rank in [(- 1), 0])):
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        logger.info('Evaluating the model on dev set of training language(en)')
        load_adapter = (((best_checkpoint + '/') + args.task_name) if (args.predict_task_adapter is None) else args.predict_task_adapter)
        logger.info(f'Task Adapter will be loaded from this path {load_adapter}')
        model.model_name = args.model_name_or_path
        (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model, load_adapter=load_adapter)
        model.to(args.device)
        (result, _) = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode='dev', prefix='debugging', lang=args.train_langs, lang2id=lang2id, lang_adapter_names=lang_adapter_names, task_name=task_name, calc_weight_step=args.calc_weight_step)
        results.update(result)
        output_eval_file = os.path.join(args.output_dir, 'eval_results.txt')
        with open(output_eval_file, 'w') as writer:
            for key in sorted(results.keys()):
                writer.write('{} = {}\n'.format(key, str(results[key])))
    if (args.do_predict and (args.local_rank in [(- 1), 0])):
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        logger.info('Evaluating the model on test set of all the languages specified')
        if (adapter_args.train_adapter or args.test_adapter):
            load_adapter = (((best_checkpoint + '/') + args.task_name) if (args.predict_task_adapter is None) else args.predict_task_adapter)
            logger.info(f'Task Adapter will be loaded from this path {load_adapter}')
            load_lang_adapter = args.predict_lang_adapter
            model.model_name = args.model_name_or_path
            (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model, load_adapter=load_adapter, load_lang_adapter=load_lang_adapter)
        model.to(args.device)
        predict_and_save(args, adapter_args, model, tokenizer, labels, lang2id, pad_token_label_id, lang_adapter_names, task_name, 'test')
    if (args.do_predict_train and (args.local_rank in [(- 1), 0])):
        logger.info('Evaluating on the train set of all specified languages')
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        if (adapter_args.train_adapter or args.test_adapter):
            load_adapter = (((best_checkpoint + '/') + args.task_name) if (args.predict_task_adapter is None) else args.predict_task_adapter)
            logger.info(f'Task Adapter will be loaded from this path {load_adapter}')
            load_lang_adapter = args.predict_lang_adapter
            model.model_name = args.model_name_or_path
            (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model, load_adapter=load_adapter, load_lang_adapter=load_lang_adapter)
        model.to(args.device)
        predict_and_save(args, adapter_args, model, tokenizer, labels, lang2id, pad_token_label_id, lang_adapter_names, task_name, 'train')
    if (args.do_predict_dev and (args.local_rank in [(- 1), 0])):
        (model, tokenizer, lang2id) = load_model(args, num_labels)
        logger.info('Evaluating on the dev sets of all the specified languages')
        if (adapter_args.train_adapter or args.test_adapter):
            load_adapter = (((best_checkpoint + '/') + args.task_name) if (args.predict_task_adapter is None) else args.predict_task_adapter)
            logger.info(f'Task Adapter will be loaded from this path {load_adapter}')
            load_lang_adapter = args.predict_lang_adapter
            model.model_name = args.model_name_or_path
            (model, lang_adapter_names, task_name) = setup_adapter(args, adapter_args, model, load_adapter=load_adapter, load_lang_adapter=load_lang_adapter)
        model.to(args.device)
        predict_and_save(args, adapter_args, model, tokenizer, labels, lang2id, pad_token_label_id, lang_adapter_names, task_name, 'dev')