codingutf8 2018 huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license split if istorchfp16availableondevicetorchdevice testargs appendfp16 with patch objectsys argv testargs runglue main result getresultstmpdir self assertgreaterequalresultevalaccuracy 0 75 def testrunclmself tmpdir self getautoremovetmpdir testargs f runclm py modelnameorpath distilgpt2 trainfile testsfixturessampletext txt validationfile testsfixturessampletext txt dotrain doeval blocksize 128 perdevicetrainbatchsize 5 perdeviceevalbatchsize 5 numtrainepochs 2 outputdir tmpdir overwriteoutputdir skipping because there are not enough batches to train the model would need a droplast to work test that configoverrides works despite the misleading dumps of default unupdated config via tokenizer split if torchdevice cpu testargs appendusecpu logger runclm logger with patch objectsys argv testargs with captureloggerlogger as cl runclm main self assertin nembd 10 cl out self assertin nhead 2 cl out def testrunmlmself tmpdir self getautoremovetmpdir testargs f runmlm py modelnameorpath distilrobertabase trainfile testsfixturessampletext txt validationfile testsfixturessampletext txt outputdir tmpdir overwriteoutputdir dotrain doeval predictionlossonly numtrainepochs1 with so little data distributed training needs more epochs to get the score on par with 01 gpu split if torchdevice cpu testargs appendusecpu with patch objectsys argv testargs runner main result getresultstmpdir self assertgreaterequalresultevalaccuracy 0 75 self assertlessresultevalloss 0 5 def testrunsquadself tmpdir self getautoremovetmpdir testargs f runqa py modelnameorpath bertbaseuncased version2withnegative trainfile testsfixturestestssamplessquadsample json validationfile testsfixturestestssamplessquadsample json outputdir tmpdir overwriteoutputdir maxsteps10 warmupsteps2 dotrain doeval learningrate2e4 perdevicetrainbatchsize2 perdeviceevalbatchsize1 split with patch objectsys argv testargs runsquadseq2seq main result getresultstmpdir self assertgreaterequalresultevalf1 30 self assertgreaterequalresultevalexact 30 def testrunswagself tmpdir self getautoremovetmpdir testargs f runswag py modelnameorpath bertbaseuncased trainfile testsfixturestestssamplesswagsample json validationfile testsfixturestestssamplesswagsample json outputdir tmpdir overwriteoutputdir maxsteps20 warmupsteps2 dotrain doeval learningrate2e4 perdevicetrainbatchsize2 perdeviceevalbatchsize1 split with patch objectsys argv testargs runsummarization main result getresultstmpdir self assertgreaterequalresultevalrouge1 10 self assertgreaterequalresultevalrouge2 2 self assertgreaterequalresultevalrougel 7 self assertgreaterequalresultevalrougelsum 7 slow def testruntranslationself tmpdir self getautoremovetmpdir testargs f runtranslation py modelnameorpath sshleiferstudentmarianenro61 sourcelang en targetlang ro trainfile testsfixturestestssampleswmt16sample json validationfile testsfixturestestssampleswmt16sample json outputdir tmpdir overwriteoutputdir maxsteps50 warmupsteps8 dotrain doeval learningrate3e3 perdevicetrainbatchsize2 perdeviceevalbatchsize1 predictwithgenerate sourcelang enxx targetlang roro split if istorchfp16availableondevicetorchdevice testargs appendfp16 with patch objectsys argv testargs runimageclassification main result getresultstmpdir self assertgreaterequalresultevalaccuracy 0 8 def testrunspeechrecognitionctcself tmpdir self getautoremovetmpdir testargs f runspeechrecognitionctc py outputdir tmpdir modelnameorpath hfinternaltestingtinyrandomwav2vec2 datasetname hfinternaltestinglibrispeechasrdummy datasetconfigname clean trainsplitname validation evalsplitname validation dotrain doeval learningrate 1e4 perdevicetrainbatchsize 2 perdeviceevalbatchsize 1 removeunusedcolumns false overwriteoutputdir true preprocessingnumworkers 16 maxsteps 10 seed 42 split if istorchfp16availableondevicetorchdevice testargs appendfp16 with patch objectsys argv testargs runspeechrecognitionctcadapter main result getresultstmpdir self asserttrueos path isfileos path jointmpdir adapter tur safetensors self assertlessresultevalloss resulttrainloss def testrunspeechrecognitionseq2seqself tmpdir self getautoremovetmpdir testargs f runspeechrecognitionseq2seq py outputdir tmpdir modelnameorpath hfinternaltestingtinyrandomspeechencoderdecoder datasetname hfinternaltestinglibrispeechasrdummy datasetconfigname clean trainsplitname validation evalsplitname validation dotrain doeval learningrate 1e4 perdevicetrainbatchsize 2 perdeviceevalbatchsize 4 removeunusedcolumns false overwriteoutputdir true preprocessingnumworkers 16 maxsteps 10 seed 42 split if istorchfp16availableondevicetorchdevice testargs appendfp16 with patch objectsys argv testargs runaudioclassification main result getresultstmpdir self assertlessresultevalloss resulttrainloss def testrunwav2vec2pretrainingself tmpdir self getautoremovetmpdir testargs f runwav2vec2pretrainingnotrainer py outputdir tmpdir modelnameorpath hfinternaltestingtinyrandomwav2vec2 datasetname hfinternaltestinglibrispeechasrdummy datasetconfignames clean datasetsplitnames validation learningrate 1e4 perdevicetrainbatchsize 4 perdeviceevalbatchsize 4 preprocessingnumworkers 16 maxtrainsteps 2 validationsplitpercentage 5 seed 42 split if istorchfp16availableondevicetorchdevice testargs appendfp16 with patch objectsys argv testargs runmae main model vitmaeforpretraining frompretrainedtmpdir self assertisnotnonemodel def testrunsemanticsegmentationself tmpdir self getautoremovetmpdir testargs f runsemanticsegmentation py outputdir tmpdir datasetname huggingfacesemanticsegmentationtestsample dotrain doeval removeunusedcolumns false overwriteoutputdir true maxsteps 10 learningrate2e4 perdevicetrainbatchsize2 perdeviceevalbatchsize1 seed 32 coding utf 8 2018 huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license run_glue py model_name_or_path distilbert base uncased output_dir tmp_dir overwrite_output_dir train_file tests fixtures tests_samples mrpc train csv validation_file tests fixtures tests_samples mrpc dev csv do_train do_eval per_device_train_batch_size 2 per_device_eval_batch_size 1 learning_rate 1e 4 max_steps 10 warmup_steps 2 seed 42 max_seq_length 128 run_clm py model_name_or_path distilgpt2 train_file tests fixtures sample_text txt validation_file tests fixtures sample_text txt do_train do_eval block_size 128 per_device_train_batch_size 5 per_device_eval_batch_size 5 num_train_epochs 2 output_dir tmp_dir overwrite_output_dir skipping because there are not enough batches to train the model would need a drop_last to work test that config_overrides works despite the misleading dumps of default un updated config via tokenizer run_clm py model_type gpt2 tokenizer_name gpt2 train_file tests fixtures sample_text txt output_dir tmp_dir config_overrides n_embd 10 n_head 2 run_mlm py model_name_or_path distilroberta base train_file tests fixtures sample_text txt validation_file tests fixtures sample_text txt output_dir tmp_dir overwrite_output_dir do_train do_eval prediction_loss_only num_train_epochs 1 with so little data distributed training needs more epochs to get the score on par with 0 1 gpu run_ner py model_name_or_path bert base uncased train_file tests fixtures tests_samples conll sample json validation_file tests fixtures tests_samples conll sample json output_dir tmp_dir overwrite_output_dir do_train do_eval warmup_steps 2 learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 2 num_train_epochs epochs seed 7 run_qa py model_name_or_path bert base uncased version_2_with_negative train_file tests fixtures tests_samples squad sample json validation_file tests fixtures tests_samples squad sample json output_dir tmp_dir overwrite_output_dir max_steps 10 warmup_steps 2 do_train do_eval learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 run_seq2seq_qa py model_name_or_path t5 small context_column context question_column question answer_column answers version_2_with_negative train_file tests fixtures tests_samples squad sample json validation_file tests fixtures tests_samples squad sample json output_dir tmp_dir overwrite_output_dir max_steps 10 warmup_steps 2 do_train do_eval learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 predict_with_generate run_swag py model_name_or_path bert base uncased train_file tests fixtures tests_samples swag sample json validation_file tests fixtures tests_samples swag sample json output_dir tmp_dir overwrite_output_dir max_steps 20 warmup_steps 2 do_train do_eval learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 run_summarization py model_name_or_path t5 small train_file tests fixtures tests_samples xsum sample json validation_file tests fixtures tests_samples xsum sample json output_dir tmp_dir overwrite_output_dir max_steps 50 warmup_steps 8 do_train do_eval learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 predict_with_generate run_translation py model_name_or_path sshleifer student_marian_en_ro_6_1 source_lang en target_lang ro train_file tests fixtures tests_samples wmt16 sample json validation_file tests fixtures tests_samples wmt16 sample json output_dir tmp_dir overwrite_output_dir max_steps 50 warmup_steps 8 do_train do_eval learning_rate 3e 3 per_device_train_batch_size 2 per_device_eval_batch_size 1 predict_with_generate source_lang en_xx target_lang ro_ro run_image_classification py output_dir tmp_dir model_name_or_path google vit base patch16 224 in21k dataset_name hf internal testing cats_vs_dogs_sample do_train do_eval learning_rate 1e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 remove_unused_columns false overwrite_output_dir true dataloader_num_workers 16 metric_for_best_model accuracy max_steps 10 train_val_split 0 1 seed 42 run_speech_recognition_ctc py output_dir tmp_dir model_name_or_path hf internal testing tiny random wav2vec2 dataset_name hf internal testing librispeech_asr_dummy dataset_config_name clean train_split_name validation eval_split_name validation do_train do_eval learning_rate 1e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 remove_unused_columns false overwrite_output_dir true preprocessing_num_workers 16 max_steps 10 seed 42 run_speech_recognition_ctc_adapter py output_dir tmp_dir model_name_or_path hf internal testing tiny random wav2vec2 dataset_name hf internal testing librispeech_asr_dummy dataset_config_name clean train_split_name validation eval_split_name validation do_train do_eval learning_rate 1e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 remove_unused_columns false overwrite_output_dir true preprocessing_num_workers 16 max_steps 10 target_language tur seed 42 run_speech_recognition_seq2seq py output_dir tmp_dir model_name_or_path hf internal testing tiny random speech encoder decoder dataset_name hf internal testing librispeech_asr_dummy dataset_config_name clean train_split_name validation eval_split_name validation do_train do_eval learning_rate 1e 4 per_device_train_batch_size 2 per_device_eval_batch_size 4 remove_unused_columns false overwrite_output_dir true preprocessing_num_workers 16 max_steps 10 seed 42 run_audio_classification py output_dir tmp_dir model_name_or_path hf internal testing tiny random wav2vec2 dataset_name anton l superb_demo dataset_config_name ks train_split_name test eval_split_name test audio_column_name audio label_column_name label do_train do_eval learning_rate 1e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 remove_unused_columns false overwrite_output_dir true num_train_epochs 10 max_steps 50 seed 42 run_wav2vec2_pretraining_no_trainer py output_dir tmp_dir model_name_or_path hf internal testing tiny random wav2vec2 dataset_name hf internal testing librispeech_asr_dummy dataset_config_names clean dataset_split_names validation learning_rate 1e 4 per_device_train_batch_size 4 per_device_eval_batch_size 4 preprocessing_num_workers 16 max_train_steps 2 validation_split_percentage 5 seed 42 run_mae py output_dir tmp_dir dataset_name hf internal testing cats_vs_dogs_sample do_train do_eval learning_rate 1e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 remove_unused_columns false overwrite_output_dir true dataloader_num_workers 16 metric_for_best_model accuracy max_steps 10 train_val_split 0 1 seed 42 run_semantic_segmentation py output_dir tmp_dir dataset_name huggingface semantic segmentation test sample do_train do_eval remove_unused_columns false overwrite_output_dir true max_steps 10 learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 seed 32
import json
import logging
import os
import sys
from unittest.mock import patch

from transformers import ViTMAEForPreTraining, Wav2Vec2ForPreTraining
from transformers.testing_utils import (
    CaptureLogger,
    TestCasePlus,
    backend_device_count,
    is_torch_fp16_available_on_device,
    slow,
    torch_device,
)


SRC_DIRS = [
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        "text-generation",
        "text-classification",
        "token-classification",
        "language-modeling",
        "multiple-choice",
        "question-answering",
        "summarization",
        "translation",
        "image-classification",
        "speech-recognition",
        "audio-classification",
        "speech-pretraining",
        "image-pretraining",
        "semantic-segmentation",
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
    import run_audio_classification
    import run_clm
    import run_generation
    import run_glue
    import run_image_classification
    import run_mae
    import run_mlm
    import run_ner
    import run_qa as run_squad
    import run_semantic_segmentation
    import run_seq2seq_qa as run_squad_seq2seq
    import run_speech_recognition_ctc
    import run_speech_recognition_ctc_adapter
    import run_speech_recognition_seq2seq
    import run_summarization
    import run_swag
    import run_translation
    import run_wav2vec2_pretraining_no_trainer


logging.basicConfig(level=logging.DEBUG)

logger = logging.getLogger()


def get_results(output_dir):
    results = {}
    path = os.path.join(output_dir, "all_results.json")
    if os.path.exists(path):
        with open(path, "r") as f:
            results = json.load(f)
    else:
        raise ValueError(f"can't find {path}")
    return results


stream_handler = logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)


class ExamplesTests(TestCasePlus):
    def test_run_glue(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_glue.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_accuracy"], 0.75)

    def test_run_clm(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if backend_device_count(torch_device) > 1:
            
            return

        if torch_device == "cpu":
            testargs.append("--use_cpu")

        with patch.object(sys, "argv", testargs):
            run_clm.main()
            result = get_results(tmp_dir)
            self.assertLess(result["perplexity"], 100)

    def test_run_clm_config_overrides(self):
        
        

        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if torch_device == "cpu":
            testargs.append("--use_cpu")

        logger = run_clm.logger
        with patch.object(sys, "argv", testargs):
            with CaptureLogger(logger) as cl:
                run_clm.main()

        self.assertIn('"n_embd": 10', cl.out)
        self.assertIn('"n_head": 2', cl.out)

    def test_run_mlm(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if torch_device == "cpu":
            testargs.append("--use_cpu")

        with patch.object(sys, "argv", testargs):
            run_mlm.main()
            result = get_results(tmp_dir)
            self.assertLess(result["perplexity"], 42)

    def test_run_ner(self):
        
        epochs = 7 if backend_device_count(torch_device) > 1 else 2

        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if torch_device == "cpu":
            testargs.append("--use_cpu")

        with patch.object(sys, "argv", testargs):
            run_ner.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_accuracy"], 0.75)
            self.assertLess(result["eval_loss"], 0.5)

    def test_run_squad(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_squad.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_f1"], 30)
            self.assertGreaterEqual(result["eval_exact"], 30)

    def test_run_squad_seq2seq(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_squad_seq2seq.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_f1"], 30)
            self.assertGreaterEqual(result["eval_exact"], 30)

    def test_run_swag(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_swag.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_accuracy"], 0.8)

    def test_generation(self):
        testargs = ["run_generation.py", "--prompt=Hello", "--length=10", "--seed=42"]

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        model_type, model_name = (
            "--model_type=gpt2",
            "--model_name_or_path=sshleifer/tiny-gpt2",
        )
        with patch.object(sys, "argv", testargs + [model_type, model_name]):
            result = run_generation.main()
            self.assertGreaterEqual(len(result[0]), 10)

    @slow
    def test_run_summarization(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_summarization.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_rouge1"], 10)
            self.assertGreaterEqual(result["eval_rouge2"], 2)
            self.assertGreaterEqual(result["eval_rougeL"], 7)
            self.assertGreaterEqual(result["eval_rougeLsum"], 7)

    @slow
    def test_run_translation(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_translation.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_bleu"], 30)

    def test_run_image_classification(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_image_classification.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_accuracy"], 0.8)

    def test_run_speech_recognition_ctc(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_speech_recognition_ctc.main()
            result = get_results(tmp_dir)
            self.assertLess(result["eval_loss"], result["train_loss"])

    def test_run_speech_recognition_ctc_adapter(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_speech_recognition_ctc_adapter.main()
            result = get_results(tmp_dir)
            self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "./adapter.tur.safetensors")))
            self.assertLess(result["eval_loss"], result["train_loss"])

    def test_run_speech_recognition_seq2seq(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_speech_recognition_seq2seq.main()
            result = get_results(tmp_dir)
            self.assertLess(result["eval_loss"], result["train_loss"])

    def test_run_audio_classification(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_audio_classification.main()
            result = get_results(tmp_dir)
            self.assertLess(result["eval_loss"], result["train_loss"])

    def test_run_wav2vec2_pretraining(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_wav2vec2_pretraining_no_trainer.main()
            model = Wav2Vec2ForPreTraining.from_pretrained(tmp_dir)
            self.assertIsNotNone(model)

    def test_run_vit_mae_pretraining(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_mae.main()
            model = ViTMAEForPreTraining.from_pretrained(tmp_dir)
            self.assertIsNotNone(model)

    def test_run_semantic_segmentation(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_semantic_segmentation.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_overall_accuracy"], 0.1)