codingutf8 2021 huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license split with patch objectsys argv testargs runflaxglue main result getresultstmpdir self assertgreaterequalresultevalaccuracy 0 75 slow def testrunclmself tmpdir self getautoremovetmpdir testargs f runclmflax py modelnameorpath distilgpt2 trainfile testsfixturessampletext txt validationfile testsfixturessampletext txt dotrain doeval blocksize 128 perdevicetrainbatchsize 4 perdeviceevalbatchsize 4 numtrainepochs 2 loggingsteps 2 evalsteps 2 outputdir tmpdir overwriteoutputdir split with patch objectsys argv testargs runsummarizationflax main result getresultstmpdir splittest self assertgreaterequalresulttestrouge1 10 self assertgreaterequalresulttestrouge2 2 self assertgreaterequalresulttestrougel 7 self assertgreaterequalresulttestrougelsum 7 slow def testrunmlmself tmpdir self getautoremovetmpdir testargs f runmlm py modelnameorpath distilrobertabase trainfile testsfixturessampletext txt validationfile testsfixturessampletext txt outputdir tmpdir overwriteoutputdir maxseqlength 128 perdevicetrainbatchsize 4 perdeviceevalbatchsize 4 loggingsteps 2 evalsteps 2 dotrain doeval numtrainepochs1 split with patch objectsys argv testargs runt5mlmflax main result getresultstmpdir self assertgreaterequalresultevalaccuracy 0 42 slow def testrunnerself with so little data distributed training needs more epochs to get the score on par with 01 gpu epochs 7 if getgpucount 1 else 2 tmpdir self getautoremovetmpdir testargs f runflaxner py modelnameorpath bertbaseuncased trainfile testsfixturestestssamplesconllsample json validationfile testsfixturestestssamplesconllsample json outputdir tmpdir overwriteoutputdir dotrain doeval warmupsteps2 learningrate2e4 loggingsteps 2 evalsteps 2 perdevicetrainbatchsize2 perdeviceevalbatchsize2 numtrainepochsepochs seed 7 split with patch objectsys argv testargs runqa main result getresultstmpdir self assertgreaterequalresultevalf1 30 self assertgreaterequalresultevalexact 30 slow def testrunflaxspeechrecognitionseq2seqself tmpdir self getautoremovetmpdir testargs f runflaxspeechrecognitionseq2seq py modelnameorpath openaiwhispertiny en datasetname hfinternaltestinglibrispeechasrdummy datasetconfig clean trainsplitname validation evalsplitname validation outputdir tmpdir overwriteoutputdir numtrainepochs2 maxtrainsamples 10 maxevalsamples 10 warmupsteps8 dotrain doeval learningrate2e4 perdevicetrainbatchsize2 perdeviceevalbatchsize1 predictwithgenerate coding utf 8 2021 huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license run_glue py model_name_or_path distilbert base uncased output_dir tmp_dir train_file tests fixtures tests_samples mrpc train csv validation_file tests fixtures tests_samples mrpc dev csv per_device_train_batch_size 2 per_device_eval_batch_size 1 learning_rate 1e 4 eval_steps 2 warmup_steps 2 seed 42 max_seq_length 128 run_clm_flax py model_name_or_path distilgpt2 train_file tests fixtures sample_text txt validation_file tests fixtures sample_text txt do_train do_eval block_size 128 per_device_train_batch_size 4 per_device_eval_batch_size 4 num_train_epochs 2 logging_steps 2 eval_steps 2 output_dir tmp_dir overwrite_output_dir run_summarization py model_name_or_path t5 small train_file tests fixtures tests_samples xsum sample json validation_file tests fixtures tests_samples xsum sample json test_file tests fixtures tests_samples xsum sample json output_dir tmp_dir overwrite_output_dir num_train_epochs 3 warmup_steps 8 do_train do_eval do_predict learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 predict_with_generate run_mlm py model_name_or_path distilroberta base train_file tests fixtures sample_text txt validation_file tests fixtures sample_text txt output_dir tmp_dir overwrite_output_dir max_seq_length 128 per_device_train_batch_size 4 per_device_eval_batch_size 4 logging_steps 2 eval_steps 2 do_train do_eval num_train_epochs 1 run_t5_mlm_flax py model_name_or_path t5 small train_file tests fixtures sample_text txt validation_file tests fixtures sample_text txt do_train do_eval max_seq_length 128 per_device_train_batch_size 4 per_device_eval_batch_size 4 num_train_epochs 2 logging_steps 2 eval_steps 2 output_dir tmp_dir overwrite_output_dir with so little data distributed training needs more epochs to get the score on par with 0 1 gpu run_flax_ner py model_name_or_path bert base uncased train_file tests fixtures tests_samples conll sample json validation_file tests fixtures tests_samples conll sample json output_dir tmp_dir overwrite_output_dir do_train do_eval warmup_steps 2 learning_rate 2e 4 logging_steps 2 eval_steps 2 per_device_train_batch_size 2 per_device_eval_batch_size 2 num_train_epochs epochs seed 7 run_qa py model_name_or_path bert base uncased version_2_with_negative train_file tests fixtures tests_samples squad sample json validation_file tests fixtures tests_samples squad sample json output_dir tmp_dir overwrite_output_dir num_train_epochs 3 warmup_steps 2 do_train do_eval logging_steps 2 eval_steps 2 learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 run_flax_speech_recognition_seq2seq py model_name_or_path openai whisper tiny en dataset_name hf internal testing librispeech_asr_dummy dataset_config clean train_split_name validation eval_split_name validation output_dir tmp_dir overwrite_output_dir num_train_epochs 2 max_train_samples 10 max_eval_samples 10 warmup_steps 8 do_train do_eval learning_rate 2e 4 per_device_train_batch_size 2 per_device_eval_batch_size 1 predict_with_generate
import argparse
import json
import logging
import os
import sys
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


SRC_DIRS = [
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        "text-classification",
        "language-modeling",
        "summarization",
        "token-classification",
        "question-answering",
        "speech-recognition",
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
    import run_clm_flax
    import run_flax_glue
    import run_flax_ner
    import run_flax_speech_recognition_seq2seq
    import run_mlm_flax
    import run_qa
    import run_summarization_flax
    import run_t5_mlm_flax


logging.basicConfig(level=logging.DEBUG)

logger = logging.getLogger()


def get_setup_file():
    parser = argparse.ArgumentParser()
    parser.add_argument("-f")
    args = parser.parse_args()
    return args.f


def get_results(output_dir, split="eval"):
    path = os.path.join(output_dir, f"{split}_results.json")
    if os.path.exists(path):
        with open(path, "r") as f:
            return json.load(f)
    raise ValueError(f"can't find {path}")


stream_handler = logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)


class ExamplesTests(TestCasePlus):
    def test_run_glue(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_flax_glue.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_accuracy"], 0.75)

    @slow
    def test_run_clm(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_clm_flax.main()
            result = get_results(tmp_dir)
            self.assertLess(result["eval_perplexity"], 100)

    @slow
    def test_run_summarization(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_summarization_flax.main()
            result = get_results(tmp_dir, split="test")
            self.assertGreaterEqual(result["test_rouge1"], 10)
            self.assertGreaterEqual(result["test_rouge2"], 2)
            self.assertGreaterEqual(result["test_rougeL"], 7)
            self.assertGreaterEqual(result["test_rougeLsum"], 7)

    @slow
    def test_run_mlm(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_mlm_flax.main()
            result = get_results(tmp_dir)
            self.assertLess(result["eval_perplexity"], 42)

    @slow
    def test_run_t5_mlm(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_t5_mlm_flax.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_accuracy"], 0.42)

    @slow
    def test_run_ner(self):
        
        epochs = 7 if get_gpu_count() > 1 else 2

        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_flax_ner.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_accuracy"], 0.75)
            self.assertGreaterEqual(result["eval_f1"], 0.3)

    @slow
    def test_run_qa(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_qa.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_f1"], 30)
            self.assertGreaterEqual(result["eval_exact"], 30)

    @slow
    def test_run_flax_speech_recognition_seq2seq(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_flax_speech_recognition_seq2seq.main()
            result = get_results(tmp_dir, split="eval")
            self.assertLessEqual(result["eval_wer"], 0.05)