usrbinenv python codingutf8 2020 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license finetuning the library models for sequence classification on glue you can also adapt this script on your own text classification task pointers for this are left as comments import logging import os import random import sys import warnings from dataclasses import dataclass field from typing import optional import datasets import evaluate import numpy as np from datasets import loaddataset import transformers from transformers import autoconfig automodelforsequenceclassification autotokenizer datacollatorwithpadding evalprediction hfargumentparser pretrainedconfig trainer trainingarguments defaultdatacollator setseed from transformers trainerutils import getlastcheckpoint from transformers utils import checkminversion sendexampletelemetry from transformers utils versions import requireversion will error if the minimal version of transformers is not installed remove at your own risks checkminversion4 36 0 dev0 requireversiondatasets1 8 0 to fix pip install r examplespytorchtextclassificationrequirements txt tasktokeys cola sentence none mnli premise hypothesis mrpc sentence1 sentence2 qnli question sentence qqp question1 question2 rte sentence1 sentence2 sst2 sentence none stsb sentence1 sentence2 wnli sentence1 sentence2 logger logging getloggername dataclass class datatrainingarguments taskname optionalstr field defaultnone metadatahelp the name of the task to train on jointasktokeys keys datasetname optionalstr field defaultnone metadatahelp the name of the dataset to use via the datasets library datasetconfigname optionalstr field defaultnone metadatahelp the configuration name of the dataset to use via the datasets library maxseqlength int field default128 metadata help the maximum total input sequence length after tokenization sequences longer than this will be truncated sequences shorter will be padded overwritecache bool field defaultfalse metadatahelp overwrite the cached preprocessed datasets or not padtomaxlength bool field defaulttrue metadata help whether to pad all samples to maxseqlength if false will pad the samples dynamically when batching to the maximum length in the batch maxtrainsamples optionalint field defaultnone metadata help for debugging purposes or quicker training truncate the number of training examples to this value if set maxevalsamples optionalint field defaultnone metadata help for debugging purposes or quicker training truncate the number of evaluation examples to this value if set maxpredictsamples optionalint field defaultnone metadata help for debugging purposes or quicker training truncate the number of prediction examples to this value if set trainfile optionalstr field defaultnone metadatahelp a csv or a json file containing the training data validationfile optionalstr field defaultnone metadatahelp a csv or a json file containing the validation data testfile optionalstr fielddefaultnone metadatahelp a csv or a json file containing the test data def postinitself if self taskname is not none self taskname self taskname lower if self taskname not in tasktokeys keys raise valueerrorunknown task you should pick one in jointasktokeys keys elif self datasetname is not none pass elif self trainfile is none or self validationfile is none raise valueerrorneed either a glue task a trainingvalidation file or a dataset name else trainextension self trainfile split 1 assert trainextension in csv json trainfile should be a csv or a json file validationextension self validationfile split 1 assert validationextension trainextension validationfile should have the same extension csv or json as trainfile dataclass class modelarguments modelnameorpath str field metadatahelp path to pretrained model or model identifier from huggingface comodels configname optionalstr field defaultnone metadatahelp pretrained config name or path if not the same as modelname tokenizername optionalstr field defaultnone metadatahelp pretrained tokenizer name or path if not the same as modelname cachedir optionalstr field defaultnone metadatahelp where do you want to store the pretrained models downloaded from huggingface co usefasttokenizer bool field defaulttrue metadatahelp whether to use one of the fast tokenizer backed by the tokenizers library or not modelrevision str field defaultmain metadatahelp the specific model version to use can be a branch name tag name or commit id token str field defaultnone metadata help the token to use as http bearer ization for remote files if not specified will use the token generated when running huggingfacecli login stored in huggingface useauthtoken bool field defaultnone metadata help the useauthtoken argument is deprecated and will be removed in v4 34 please use token instead trustremotecode bool field defaultfalse metadata help whether or not to allow for custom models defined on the hub in their own modeling files this option should only be set to true for repositories you trust and in which you have read the code as it will execute code present on the hub on your local machine ignoremismatchedsizes bool field defaultfalse metadatahelp will enable to load a pretrained model whose head dimensions are different def main see all possible arguments in srctransformerstrainingargs py or by passing the help flag to this script we now keep distinct sets of args for a cleaner separation of concerns parser hfargumentparsermodelarguments datatrainingarguments trainingarguments if lensys argv 2 and sys argv1 endswith json if we pass only one argument to the script and it s the path to a json file let s parse it to get our arguments modelargs dataargs trainingargs parser parsejsonfilejsonfileos path abspathsys argv1 else modelargs dataargs trainingargs parser parseargsintodataclasses if modelargs useauthtoken is not none warnings warn the useauthtoken argument is deprecated and will be removed in v4 34 please use token instead futurewarning if modelargs token is not none raise valueerrortoken and useauthtoken are both specified please set only the argument token modelargs token modelargs useauthtoken sending telemetry tracking the example usage helps us better allocate resources to maintain them the information sent is the one passed as arguments along with your pythonpytorch versions sendexampletelemetryrunglue modelargs dataargs setup logging logging basicconfig formatasctimes levelnames names messages datefmtmdy h m s handlerslogging streamhandlersys stdout if trainingargs shouldlog the default of trainingargs loglevel is passive so we set log level at info here to have that default transformers utils logging setverbosityinfo loglevel trainingargs getprocessloglevel logger setlevelloglevel datasets utils logging setverbosityloglevel transformers utils logging setverbosityloglevel transformers utils logging enabledefaulthandler transformers utils logging enableexplicitformat log on each process the small summary logger warning fprocess rank trainingargs localrank device trainingargs device ngpu trainingargs ngpu fdistributed training trainingargs parallelmode value distributed 16bits training trainingargs fp16 logger infoftrainingevaluation parameters trainingargs detecting last checkpoint lastcheckpoint none if os path isdirtrainingargs outputdir and trainingargs dotrain and not trainingargs overwriteoutputdir lastcheckpoint getlastcheckpointtrainingargs outputdir if lastcheckpoint is none and lenos listdirtrainingargs outputdir 0 raise valueerror foutput directory trainingargs outputdir already exists and is not empty use overwriteoutputdir to overcome elif lastcheckpoint is not none and trainingargs resumefromcheckpoint is none logger info fcheckpoint detected resuming training at lastcheckpoint to avoid this behavior change the outputdir or add overwriteoutputdir to train from scratch set seed before initializing model setseedtrainingargs seed get the datasets you can either provide your own csvjson training and evaluation files see below or specify a glue benchmark task the dataset will be downloaded automatically from the datasets hub for csvjson files this script will use as labels the column called label and as pair of sentences the sentences in columns called sentence1 and sentence2 if such column exists or the first two columns not named label if at least two columns are provided if the csvsjsons contain only one nonlabel column the script does single sentence classification on this single column you can easily tweak this behavior see below in distributed training the loaddataset function guarantee that only one local process can concurrently download the dataset if dataargs taskname is not none downloading and loading a dataset from the hub rawdatasets loaddataset glue dataargs taskname cachedirmodelargs cachedir tokenmodelargs token elif dataargs datasetname is not none downloading and loading a dataset from the hub rawdatasets loaddataset dataargs datasetname dataargs datasetconfigname cachedirmodelargs cachedir tokenmodelargs token else loading a dataset from your local files csvjson training and evaluation files are needed datafiles train dataargs trainfile validation dataargs validationfile get the test dataset you can provide your own csvjson test file see below when you use dopredict without specifying a glue benchmark task if trainingargs dopredict if dataargs testfile is not none trainextension dataargs trainfile split 1 testextension dataargs testfile split 1 assert testextension trainextension testfile should have the same extension csv or json as trainfile datafilestest dataargs testfile else raise valueerrorneed either a glue task or a test file for dopredict for key in datafiles keys logger infofload a local file for key datafileskey if dataargs trainfile endswith csv loading a dataset from local csv files rawdatasets loaddataset csv datafilesdatafiles cachedirmodelargs cachedir tokenmodelargs token else loading a dataset from local json files rawdatasets loaddataset json datafilesdatafiles cachedirmodelargs cachedir tokenmodelargs token see more about loading any type of standard or custom dataset at https huggingface codocsdatasetsloadingdatasets labels if dataargs taskname is not none isregression dataargs taskname stsb if not isregression labellist rawdatasetstrain featureslabel names numlabels lenlabellist else numlabels 1 else trying to have good defaults here don t hesitate to tweak to your needs isregression rawdatasetstrain featureslabel dtype in float32 float64 if isregression numlabels 1 else a useful fast method https huggingface codocsdatasetspackagereferencemainclassesdatasets dataset unique labellist rawdatasetstrain uniquelabel labellist sort let s sort it for determinism numlabels lenlabellist load pretrained model and tokenizer in distributed training the frompretrained methods guarantee that only one local process can concurrently download model vocab config autoconfig frompretrained modelargs configname if modelargs configname else modelargs modelnameorpath numlabelsnumlabels finetuningtaskdataargs taskname cachedirmodelargs cachedir revisionmodelargs modelrevision tokenmodelargs token trustremotecodemodelargs trustremotecode tokenizer autotokenizer frompretrained modelargs tokenizername if modelargs tokenizername else modelargs modelnameorpath cachedirmodelargs cachedir usefastmodelargs usefasttokenizer revisionmodelargs modelrevision tokenmodelargs token trustremotecodemodelargs trustremotecode model automodelforsequenceclassification frompretrained modelargs modelnameorpath fromtfbool ckpt in modelargs modelnameorpath configconfig cachedirmodelargs cachedir revisionmodelargs modelrevision tokenmodelargs token trustremotecodemodelargs trustremotecode ignoremismatchedsizesmodelargs ignoremismatchedsizes preprocessing the rawdatasets if dataargs taskname is not none sentence1key sentence2key tasktokeysdataargs taskname else again we try to have some nice defaults but don t hesitate to tweak to your use case nonlabelcolumnnames name for name in rawdatasetstrain columnnames if name label if sentence1 in nonlabelcolumnnames and sentence2 in nonlabelcolumnnames sentence1key sentence2key sentence1 sentence2 else if lennonlabelcolumnnames 2 sentence1key sentence2key nonlabelcolumnnames 2 else sentence1key sentence2key nonlabelcolumnnames0 none padding strategy if dataargs padtomaxlength padding maxlength else we will pad later dynamically at batch creation to the max sequence length in each batch padding false some models have set the order of the labels to use so let s make sure we do use it labeltoid none if model config label2id pretrainedconfignumlabelsnumlabels label2id and dataargs taskname is not none and not isregression some have all caps in their config some don t labelnametoid k lower v for k v in model config label2id items if sortedlabelnametoid keys sortedlabellist labeltoid i intlabelnametoidlabellisti for i in rangenumlabels else logger warning your model seems to have been trained with labels but they don t match the dataset fmodel labels sortedlabelnametoid keys dataset labels sortedlabellist nignoring the model labels as a result elif dataargs taskname is none and not isregression labeltoid v i for i v in enumeratelabellist if labeltoid is not none model config label2id labeltoid model config id2label id label for label id in config label2id items elif dataargs taskname is not none and not isregression model config label2id l i for i l in enumeratelabellist model config id2label id label for label id in config label2id items if dataargs maxseqlength tokenizer modelmaxlength logger warning fthe maxseqlength passed dataargs maxseqlength is larger than the maximum length for the fmodel tokenizer modelmaxlength using maxseqlengthtokenizer modelmaxlength maxseqlength mindataargs maxseqlength tokenizer modelmaxlength def preprocessfunctionexamples tokenize the texts args examplessentence1key if sentence2key is none else examplessentence1key examplessentence2key result tokenizerargs paddingpadding maxlengthmaxseqlength truncationtrue map labels to ids not necessary for glue tasks if labeltoid is not none and label in examples resultlabel labeltoidl if l 1 else 1 for l in exampleslabel return result with trainingargs mainprocessfirstdescdataset map preprocessing rawdatasets rawdatasets map preprocessfunction batchedtrue loadfromcachefilenot dataargs overwritecache descrunning tokenizer on dataset if trainingargs dotrain if train not in rawdatasets raise valueerrordotrain requires a train dataset traindataset rawdatasetstrain if dataargs maxtrainsamples is not none maxtrainsamples minlentraindataset dataargs maxtrainsamples traindataset traindataset selectrangemaxtrainsamples if trainingargs doeval if validation not in rawdatasets and validationmatched not in rawdatasets raise valueerrordoeval requires a validation dataset evaldataset rawdatasetsvalidationmatched if dataargs taskname mnli else validation if dataargs maxevalsamples is not none maxevalsamples minlenevaldataset dataargs maxevalsamples evaldataset evaldataset selectrangemaxevalsamples if trainingargs dopredict or dataargs taskname is not none or dataargs testfile is not none if test not in rawdatasets and testmatched not in rawdatasets raise valueerrordopredict requires a test dataset predictdataset rawdatasetstestmatched if dataargs taskname mnli else test if dataargs maxpredictsamples is not none maxpredictsamples minlenpredictdataset dataargs maxpredictsamples predictdataset predictdataset selectrangemaxpredictsamples log a few random samples from the training set if trainingargs dotrain for index in random samplerangelentraindataset 3 logger infofsample index of the training set traindatasetindex get the metric function if dataargs taskname is not none metric evaluate loadglue dataargs taskname elif isregression metric evaluate loadmse else metric evaluate loadaccuracy you can define your custom computemetrics function it takes an evalprediction object a namedtuple with a predictions and labelids field and has to return a dictionary string to float def computemetricsp evalprediction preds p predictions0 if isinstancep predictions tuple else p predictions preds np squeezepreds if isregression else np argmaxpreds axis1 result metric computepredictionspreds referencesp labelids if lenresult 1 resultcombinedscore np meanlistresult values item return result data collator will default to datacollatorwithpadding when the tokenizer is passed to trainer so we change it if we already did the padding if dataargs padtomaxlength datacollator defaultdatacollator elif trainingargs fp16 datacollator datacollatorwithpaddingtokenizer padtomultipleof8 else datacollator none initialize our trainer trainer trainer modelmodel argstrainingargs traindatasettraindataset if trainingargs dotrain else none evaldatasetevaldataset if trainingargs doeval else none computemetricscomputemetrics tokenizertokenizer datacollatordatacollator training if trainingargs dotrain checkpoint none if trainingargs resumefromcheckpoint is not none checkpoint trainingargs resumefromcheckpoint elif lastcheckpoint is not none checkpoint lastcheckpoint trainresult trainer trainresumefromcheckpointcheckpoint metrics trainresult metrics maxtrainsamples dataargs maxtrainsamples if dataargs maxtrainsamples is not none else lentraindataset metricstrainsamples minmaxtrainsamples lentraindataset trainer savemodel saves the tokenizer too for easy upload trainer logmetricstrain metrics trainer savemetricstrain metrics trainer savestate evaluation if trainingargs doeval logger info evaluate loop to handle mnli double evaluation matched mismatched tasks dataargs taskname evaldatasets evaldataset if dataargs taskname mnli tasks appendmnlimm validmmdataset rawdatasetsvalidationmismatched if dataargs maxevalsamples is not none maxevalsamples minlenvalidmmdataset dataargs maxevalsamples validmmdataset validmmdataset selectrangemaxevalsamples evaldatasets appendvalidmmdataset combined for evaldataset task in zipevaldatasets tasks metrics trainer evaluateevaldatasetevaldataset maxevalsamples dataargs maxevalsamples if dataargs maxevalsamples is not none else lenevaldataset metricsevalsamples minmaxevalsamples lenevaldataset if task mnlimm metrics k mm v for k v in metrics items if task is not none and mnli in task combined updatemetrics trainer logmetricseval metrics trainer savemetricseval combined if task is not none and mnli in task else metrics if trainingargs dopredict logger info predict loop to handle mnli double evaluation matched mismatched tasks dataargs taskname predictdatasets predictdataset if dataargs taskname mnli tasks appendmnlimm predictdatasets appendrawdatasetstestmismatched for predictdataset task in zippredictdatasets tasks removing the label columns because it contains 1 and trainer won t like that predictdataset predictdataset removecolumnslabel predictions trainer predictpredictdataset metrickeyprefixpredict predictions predictions np squeezepredictions if isregression else np argmaxpredictions axis1 outputpredictfile os path jointrainingargs outputdir fpredictresultstask txt if trainer isworldprocesszero with openoutputpredictfile w as writer logger infof predict results task writer writeindextpredictionn for index item in enumeratepredictions if isregression writer writefindextitem 3 3fn else item labellistitem writer writefindextitemn kwargs finetunedfrom modelargs modelnameorpath tasks textclassification if dataargs taskname is not none kwargslanguage en kwargsdatasettags glue kwargsdatasetargs dataargs taskname kwargsdataset fglue dataargs taskname upper if trainingargs pushtohub trainer pushtohubkwargs else trainer createmodelcardkwargs def mpfnindex for xlaspawn tpus main if name main main usr bin env python coding utf 8 2020 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license finetuning the library models for sequence classification on glue you can also adapt this script on your own text classification task pointers for this are left as comments will error if the minimal version of transformers is not installed remove at your own risks arguments pertaining to what data we are going to input our model for training and eval using hfargumentparser we can turn this class into argparse arguments to be able to specify them on the command line arguments pertaining to which model config tokenizer we are going to fine tune from see all possible arguments in src transformers training_args py or by passing the help flag to this script we now keep distinct sets of args for a cleaner separation of concerns if we pass only one argument to the script and it s the path to a json file let s parse it to get our arguments sending telemetry tracking the example usage helps us better allocate resources to maintain them the information sent is the one passed as arguments along with your python pytorch versions setup logging the default of training_args log_level is passive so we set log level at info here to have that default log on each process the small summary detecting last checkpoint set seed before initializing model get the datasets you can either provide your own csv json training and evaluation files see below or specify a glue benchmark task the dataset will be downloaded automatically from the datasets hub for csv json files this script will use as labels the column called label and as pair of sentences the sentences in columns called sentence1 and sentence2 if such column exists or the first two columns not named label if at least two columns are provided if the csvs jsons contain only one non label column the script does single sentence classification on this single column you can easily tweak this behavior see below in distributed training the load_dataset function guarantee that only one local process can concurrently download the dataset downloading and loading a dataset from the hub downloading and loading a dataset from the hub loading a dataset from your local files csv json training and evaluation files are needed get the test dataset you can provide your own csv json test file see below when you use do_predict without specifying a glue benchmark task loading a dataset from local csv files loading a dataset from local json files see more about loading any type of standard or custom dataset at https huggingface co docs datasets loading_datasets labels trying to have good defaults here don t hesitate to tweak to your needs a useful fast method https huggingface co docs datasets package_reference main_classes datasets dataset unique let s sort it for determinism load pretrained model and tokenizer in distributed training the from_pretrained methods guarantee that only one local process can concurrently download model vocab preprocessing the raw_datasets again we try to have some nice defaults but don t hesitate to tweak to your use case padding strategy we will pad later dynamically at batch creation to the max sequence length in each batch some models have set the order of the labels to use so let s make sure we do use it some have all caps in their config some don t tokenize the texts map labels to ids not necessary for glue tasks log a few random samples from the training set get the metric function you can define your custom compute_metrics function it takes an evalprediction object a namedtuple with a predictions and label_ids field and has to return a dictionary string to float data collator will default to datacollatorwithpadding when the tokenizer is passed to trainer so we change it if we already did the padding initialize our trainer training saves the tokenizer too for easy upload evaluation loop to handle mnli double evaluation matched mis matched loop to handle mnli double evaluation matched mis matched removing the label columns because it contains 1 and trainer won t like that for xla_spawn tpus
import logging
import os
import random
import sys
import warnings
from dataclasses import dataclass, field
from typing import Optional

import datasets
import evaluate
import numpy as np
from datasets import load_dataset

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    PretrainedConfig,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version, send_example_telemetry
from transformers.utils.versions import require_version



check_min_version("4.36.0.dev0")

require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt")

task_to_keys = {
    "cola": ("sentence", None),
    "mnli": ("premise", "hypothesis"),
    "mrpc": ("sentence1", "sentence2"),
    "qnli": ("question", "sentence"),
    "qqp": ("question1", "question2"),
    "rte": ("sentence1", "sentence2"),
    "sst2": ("sentence", None),
    "stsb": ("sentence1", "sentence2"),
    "wnli": ("sentence1", "sentence2"),
}

logger = logging.getLogger(__name__)


@dataclass
class DataTrainingArguments:
    

    task_name: Optional[str] = field(
        default=None,
        metadata={"help": "The name of the task to train on: " + ", ".join(task_to_keys.keys())},
    )
    dataset_name: Optional[str] = field(
        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
    )
    dataset_config_name: Optional[str] = field(
        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
    )
    max_seq_length: int = field(
        default=128,
        metadata={
            "help": (
                "The maximum total input sequence length after tokenization. Sequences longer "
                "than this will be truncated, sequences shorter will be padded."
            )
        },
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."}
    )
    pad_to_max_length: bool = field(
        default=True,
        metadata={
            "help": (
                "Whether to pad all samples to `max_seq_length`. "
                "If False, will pad the samples dynamically when batching to the maximum length in the batch."
            )
        },
    )
    max_train_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": (
                "For debugging purposes or quicker training, truncate the number of training examples to this "
                "value if set."
            )
        },
    )
    max_eval_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": (
                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
                "value if set."
            )
        },
    )
    max_predict_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": (
                "For debugging purposes or quicker training, truncate the number of prediction examples to this "
                "value if set."
            )
        },
    )
    train_file: Optional[str] = field(
        default=None, metadata={"help": "A csv or a json file containing the training data."}
    )
    validation_file: Optional[str] = field(
        default=None, metadata={"help": "A csv or a json file containing the validation data."}
    )
    test_file: Optional[str] = field(default=None, metadata={"help": "A csv or a json file containing the test data."})

    def __post_init__(self):
        if self.task_name is not None:
            self.task_name = self.task_name.lower()
            if self.task_name not in task_to_keys.keys():
                raise ValueError("Unknown task, you should pick one in " + ",".join(task_to_keys.keys()))
        elif self.dataset_name is not None:
            pass
        elif self.train_file is None or self.validation_file is None:
            raise ValueError("Need either a GLUE task, a training/validation file or a dataset name.")
        else:
            train_extension = self.train_file.split(".")[-1]
            assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file."
            validation_extension = self.validation_file.split(".")[-1]
            assert (
                validation_extension == train_extension
            ), "`validation_file` should have the same extension (csv or json) as `train_file`."


@dataclass
class ModelArguments:
    

    model_name_or_path: str = field(
        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
    )
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
    )
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
        default=None,
        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
    )
    use_fast_tokenizer: bool = field(
        default=True,
        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
    )
    model_revision: str = field(
        default="main",
        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
    )
    token: str = field(
        default=None,
        metadata={
            "help": (
                "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
                "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
            )
        },
    )
    use_auth_token: bool = field(
        default=None,
        metadata={
            "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead."
        },
    )
    trust_remote_code: bool = field(
        default=False,
        metadata={
            "help": (
                "Whether or not to allow for custom models defined on the Hub in their own modeling files. This option"
                "should only be set to `True` for repositories you trust and in which you have read the code, as it will "
                "execute code present on the Hub on your local machine."
            )
        },
    )
    ignore_mismatched_sizes: bool = field(
        default=False,
        metadata={"help": "Will enable to load a pretrained model whose head dimensions are different."},
    )


def main():
    
    
    

    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
        
        
        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    if model_args.use_auth_token is not None:
        warnings.warn(
            "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead.",
            FutureWarning,
        )
        if model_args.token is not None:
            raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
        model_args.token = model_args.use_auth_token

    
    
    send_example_telemetry("run_glue", model_args, data_args)

    
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        handlers=[logging.StreamHandler(sys.stdout)],
    )

    if training_args.should_log:
        
        transformers.utils.logging.set_verbosity_info()

    log_level = training_args.get_process_log_level()
    logger.setLevel(log_level)
    datasets.utils.logging.set_verbosity(log_level)
    transformers.utils.logging.set_verbosity(log_level)
    transformers.utils.logging.enable_default_handler()
    transformers.utils.logging.enable_explicit_format()

    
    logger.warning(
        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, "
        + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}"
    )
    logger.info(f"Training/evaluation parameters {training_args}")

    
    last_checkpoint = None
    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
        last_checkpoint = get_last_checkpoint(training_args.output_dir)
        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
            raise ValueError(
                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
                "Use --overwrite_output_dir to overcome."
            )
        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
            logger.info(
                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
            )

    
    set_seed(training_args.seed)

    
    
    
    
    
    
    
    
    
    
    
    
    if data_args.task_name is not None:
        
        raw_datasets = load_dataset(
            "glue",
            data_args.task_name,
            cache_dir=model_args.cache_dir,
            token=model_args.token,
        )
    elif data_args.dataset_name is not None:
        
        raw_datasets = load_dataset(
            data_args.dataset_name,
            data_args.dataset_config_name,
            cache_dir=model_args.cache_dir,
            token=model_args.token,
        )
    else:
        
        
        data_files = {"train": data_args.train_file, "validation": data_args.validation_file}

        
        
        if training_args.do_predict:
            if data_args.test_file is not None:
                train_extension = data_args.train_file.split(".")[-1]
                test_extension = data_args.test_file.split(".")[-1]
                assert (
                    test_extension == train_extension
                ), "`test_file` should have the same extension (csv or json) as `train_file`."
                data_files["test"] = data_args.test_file
            else:
                raise ValueError("Need either a GLUE task or a test file for `do_predict`.")

        for key in data_files.keys():
            logger.info(f"load a local file for {key}: {data_files[key]}")

        if data_args.train_file.endswith(".csv"):
            
            raw_datasets = load_dataset(
                "csv",
                data_files=data_files,
                cache_dir=model_args.cache_dir,
                token=model_args.token,
            )
        else:
            
            raw_datasets = load_dataset(
                "json",
                data_files=data_files,
                cache_dir=model_args.cache_dir,
                token=model_args.token,
            )
    
    

    
    if data_args.task_name is not None:
        is_regression = data_args.task_name == "stsb"
        if not is_regression:
            label_list = raw_datasets["train"].features["label"].names
            num_labels = len(label_list)
        else:
            num_labels = 1
    else:
        
        is_regression = raw_datasets["train"].features["label"].dtype in ["float32", "float64"]
        if is_regression:
            num_labels = 1
        else:
            
            
            label_list = raw_datasets["train"].unique("label")
            label_list.sort()  
            num_labels = len(label_list)

    
    
    
    
    config = AutoConfig.from_pretrained(
        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
        num_labels=num_labels,
        finetuning_task=data_args.task_name,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        token=model_args.token,
        trust_remote_code=model_args.trust_remote_code,
    )
    tokenizer = AutoTokenizer.from_pretrained(
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
        cache_dir=model_args.cache_dir,
        use_fast=model_args.use_fast_tokenizer,
        revision=model_args.model_revision,
        token=model_args.token,
        trust_remote_code=model_args.trust_remote_code,
    )
    model = AutoModelForSequenceClassification.from_pretrained(
        model_args.model_name_or_path,
        from_tf=bool(".ckpt" in model_args.model_name_or_path),
        config=config,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        token=model_args.token,
        trust_remote_code=model_args.trust_remote_code,
        ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,
    )

    
    if data_args.task_name is not None:
        sentence1_key, sentence2_key = task_to_keys[data_args.task_name]
    else:
        
        non_label_column_names = [name for name in raw_datasets["train"].column_names if name != "label"]
        if "sentence1" in non_label_column_names and "sentence2" in non_label_column_names:
            sentence1_key, sentence2_key = "sentence1", "sentence2"
        else:
            if len(non_label_column_names) >= 2:
                sentence1_key, sentence2_key = non_label_column_names[:2]
            else:
                sentence1_key, sentence2_key = non_label_column_names[0], None

    
    if data_args.pad_to_max_length:
        padding = "max_length"
    else:
        
        padding = False

    
    label_to_id = None
    if (
        model.config.label2id != PretrainedConfig(num_labels=num_labels).label2id
        and data_args.task_name is not None
        and not is_regression
    ):
        
        label_name_to_id = {k.lower(): v for k, v in model.config.label2id.items()}
        if sorted(label_name_to_id.keys()) == sorted(label_list):
            label_to_id = {i: int(label_name_to_id[label_list[i]]) for i in range(num_labels)}
        else:
            logger.warning(
                "Your model seems to have been trained with labels, but they don't match the dataset: ",
                f"model labels: {sorted(label_name_to_id.keys())}, dataset labels: {sorted(label_list)}."
                "\nIgnoring the model labels as a result.",
            )
    elif data_args.task_name is None and not is_regression:
        label_to_id = {v: i for i, v in enumerate(label_list)}

    if label_to_id is not None:
        model.config.label2id = label_to_id
        model.config.id2label = {id: label for label, id in config.label2id.items()}
    elif data_args.task_name is not None and not is_regression:
        model.config.label2id = {l: i for i, l in enumerate(label_list)}
        model.config.id2label = {id: label for label, id in config.label2id.items()}

    if data_args.max_seq_length > tokenizer.model_max_length:
        logger.warning(
            f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the "
            f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
        )
    max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)

    def preprocess_function(examples):
        
        args = (
            (examples[sentence1_key],) if sentence2_key is None else (examples[sentence1_key], examples[sentence2_key])
        )
        result = tokenizer(*args, padding=padding, max_length=max_seq_length, truncation=True)

        
        if label_to_id is not None and "label" in examples:
            result["label"] = [(label_to_id[l] if l != -1 else -1) for l in examples["label"]]
        return result

    with training_args.main_process_first(desc="dataset map pre-processing"):
        raw_datasets = raw_datasets.map(
            preprocess_function,
            batched=True,
            load_from_cache_file=not data_args.overwrite_cache,
            desc="Running tokenizer on dataset",
        )
    if training_args.do_train:
        if "train" not in raw_datasets:
            raise ValueError("--do_train requires a train dataset")
        train_dataset = raw_datasets["train"]
        if data_args.max_train_samples is not None:
            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
            train_dataset = train_dataset.select(range(max_train_samples))

    if training_args.do_eval:
        if "validation" not in raw_datasets and "validation_matched" not in raw_datasets:
            raise ValueError("--do_eval requires a validation dataset")
        eval_dataset = raw_datasets["validation_matched" if data_args.task_name == "mnli" else "validation"]
        if data_args.max_eval_samples is not None:
            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
            eval_dataset = eval_dataset.select(range(max_eval_samples))

    if training_args.do_predict or data_args.task_name is not None or data_args.test_file is not None:
        if "test" not in raw_datasets and "test_matched" not in raw_datasets:
            raise ValueError("--do_predict requires a test dataset")
        predict_dataset = raw_datasets["test_matched" if data_args.task_name == "mnli" else "test"]
        if data_args.max_predict_samples is not None:
            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)
            predict_dataset = predict_dataset.select(range(max_predict_samples))

    
    if training_args.do_train:
        for index in random.sample(range(len(train_dataset)), 3):
            logger.info(f"Sample {index} of the training set: {train_dataset[index]}.")

    
    if data_args.task_name is not None:
        metric = evaluate.load("glue", data_args.task_name)
    elif is_regression:
        metric = evaluate.load("mse")
    else:
        metric = evaluate.load("accuracy")

    
    
    def compute_metrics(p: EvalPrediction):
        preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions
        preds = np.squeeze(preds) if is_regression else np.argmax(preds, axis=1)
        result = metric.compute(predictions=preds, references=p.label_ids)
        if len(result) > 1:
            result["combined_score"] = np.mean(list(result.values())).item()
        return result

    
    
    if data_args.pad_to_max_length:
        data_collator = default_data_collator
    elif training_args.fp16:
        data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8)
    else:
        data_collator = None

    
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset if training_args.do_train else None,
        eval_dataset=eval_dataset if training_args.do_eval else None,
        compute_metrics=compute_metrics,
        tokenizer=tokenizer,
        data_collator=data_collator,
    )

    
    if training_args.do_train:
        checkpoint = None
        if training_args.resume_from_checkpoint is not None:
            checkpoint = training_args.resume_from_checkpoint
        elif last_checkpoint is not None:
            checkpoint = last_checkpoint
        train_result = trainer.train(resume_from_checkpoint=checkpoint)
        metrics = train_result.metrics
        max_train_samples = (
            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
        )
        metrics["train_samples"] = min(max_train_samples, len(train_dataset))

        trainer.save_model()  

        trainer.log_metrics("train", metrics)
        trainer.save_metrics("train", metrics)
        trainer.save_state()

    
    if training_args.do_eval:
        logger.info("*** Evaluate ***")

        
        tasks = [data_args.task_name]
        eval_datasets = [eval_dataset]
        if data_args.task_name == "mnli":
            tasks.append("mnli-mm")
            valid_mm_dataset = raw_datasets["validation_mismatched"]
            if data_args.max_eval_samples is not None:
                max_eval_samples = min(len(valid_mm_dataset), data_args.max_eval_samples)
                valid_mm_dataset = valid_mm_dataset.select(range(max_eval_samples))
            eval_datasets.append(valid_mm_dataset)
            combined = {}

        for eval_dataset, task in zip(eval_datasets, tasks):
            metrics = trainer.evaluate(eval_dataset=eval_dataset)

            max_eval_samples = (
                data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
            )
            metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))

            if task == "mnli-mm":
                metrics = {k + "_mm": v for k, v in metrics.items()}
            if task is not None and "mnli" in task:
                combined.update(metrics)

            trainer.log_metrics("eval", metrics)
            trainer.save_metrics("eval", combined if task is not None and "mnli" in task else metrics)

    if training_args.do_predict:
        logger.info("*** Predict ***")

        
        tasks = [data_args.task_name]
        predict_datasets = [predict_dataset]
        if data_args.task_name == "mnli":
            tasks.append("mnli-mm")
            predict_datasets.append(raw_datasets["test_mismatched"])

        for predict_dataset, task in zip(predict_datasets, tasks):
            
            predict_dataset = predict_dataset.remove_columns("label")
            predictions = trainer.predict(predict_dataset, metric_key_prefix="predict").predictions
            predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1)

            output_predict_file = os.path.join(training_args.output_dir, f"predict_results_{task}.txt")
            if trainer.is_world_process_zero():
                with open(output_predict_file, "w") as writer:
                    logger.info(f"***** Predict results {task} *****")
                    writer.write("index\tprediction\n")
                    for index, item in enumerate(predictions):
                        if is_regression:
                            writer.write(f"{index}\t{item:3.3f}\n")
                        else:
                            item = label_list[item]
                            writer.write(f"{index}\t{item}\n")

    kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-classification"}
    if data_args.task_name is not None:
        kwargs["language"] = "en"
        kwargs["dataset_tags"] = "glue"
        kwargs["dataset_args"] = data_args.task_name
        kwargs["dataset"] = f"GLUE {data_args.task_name.upper()}"

    if training_args.push_to_hub:
        trainer.push_to_hub(**kwargs)
    else:
        trainer.create_model_card(**kwargs)


def _mp_fn(index):
    
    main()


if __name__ == "__main__":
    main()