2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license split with patch objectsys argv testargs rungenerate assert pathoutputfilename exists os removepathoutputfilename test one model to quickly noslow catch simple problems and do an extensive testing of functionality with multiple models as slow separately def testrunevalself self runevaltestert5tiny any extra models should go into the list here can be slow parameterized expandbarttiny mbarttiny slow def testrunevalslowself model self runevaltestermodel testing with 2 models to validate 1 translation t5 2 summarization mbart parameterized expandt5tiny mbarttiny slow def testrunevalsearchself model inputfilename pathself getautoremovetmpdir utestinput source outputfilename inputfilename parent utestoutput txt assert not outputfilename exists text en machine learning is great isn t it i like to eat bananas tomorrow is another great day de maschinelles lernen ist groartig oder ich esse gerne bananen morgen ist wieder ein toller tag tmpdir pathself getautoremovetmpdir scorepath strtmpdir scores json referencepath strtmpdir val target dumparticlesinputfilename texten dumparticlesreferencepath textde task translationentode if model t5tiny else summarization testargs f runevalsearch py model strinputfilename stroutputfilename scorepath scorepath referencepath referencepath task task 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license remove noisy download output from tracebacks run_eval_search py model input_file_name output_file_name score_path score_path task task num_beams 2 length_penalty 2 0 os remove path output_file_name test one model to quickly no slow catch simple problems and do an extensive testing of functionality with multiple models as slow separately any extra models should go into the list here can be slow testing with 2 models to validate 1 translation t5 2 summarization mbart run_eval_search py model str input_file_name str output_file_name score_path score_path reference_path reference_path task task
import logging
import os
import sys
from pathlib import Path
from unittest.mock import patch

from parameterized import parameterized
from run_eval import run_generate
from run_eval_search import run_search

from transformers.testing_utils import CaptureStdout, TestCasePlus, slow
from utils import ROUGE_KEYS


logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()


def _dump_articles(path: Path, articles: list):
    content = "\n".join(articles)
    Path(path).open("w").writelines(content)


T5_TINY = "patrickvonplaten/t5-tiny-random"
BART_TINY = "sshleifer/bart-tiny-random"
MBART_TINY = "sshleifer/tiny-mbart"

stream_handler = logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
logging.disable(logging.CRITICAL)  


class TestTheRest(TestCasePlus):
    def run_eval_tester(self, model):
        input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
        output_file_name = input_file_name.parent / "utest_output.txt"
        assert not output_file_name.exists()
        articles = [" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County."]
        _dump_articles(input_file_name, articles)

        score_path = str(Path(self.get_auto_remove_tmp_dir()) / "scores.json")
        task = "translation_en_to_de" if model == T5_TINY else "summarization"
        testargs = f.split()

        with patch.object(sys, "argv", testargs):
            run_generate()
            assert Path(output_file_name).exists()
            

    
    
    def test_run_eval(self):
        self.run_eval_tester(T5_TINY)

    
    @parameterized.expand([BART_TINY, MBART_TINY])
    @slow
    def test_run_eval_slow(self, model):
        self.run_eval_tester(model)

    
    @parameterized.expand([T5_TINY, MBART_TINY])
    @slow
    def test_run_eval_search(self, model):
        input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
        output_file_name = input_file_name.parent / "utest_output.txt"
        assert not output_file_name.exists()

        text = {
            "en": ["Machine learning is great, isn't it?", "I like to eat bananas", "Tomorrow is another great day!"],
            "de": [
                "Maschinelles Lernen ist großartig, oder?",
                "Ich esse gerne Bananen",
                "Morgen ist wieder ein toller Tag!",
            ],
        }

        tmp_dir = Path(self.get_auto_remove_tmp_dir())
        score_path = str(tmp_dir / "scores.json")
        reference_path = str(tmp_dir / "val.target")
        _dump_articles(input_file_name, text["en"])
        _dump_articles(reference_path, text["de"])
        task = "translation_en_to_de" if model == T5_TINY else "summarization"
        testargs = f.split()
        testargs.extend(["--search", "num_beams=1:2 length_penalty=0.9:1.0"])

        with patch.object(sys, "argv", testargs):
            with CaptureStdout() as cs:
                run_search()
            expected_strings = [" num_beams | length_penalty", model, "Best score args"]
            un_expected_strings = ["Info"]
            if "translation" in task:
                expected_strings.append("bleu")
            else:
                expected_strings.extend(ROUGE_KEYS)
            for w in expected_strings:
                assert w in cs.out
            for w in un_expected_strings:
                assert w not in cs.out
            assert Path(output_file_name).exists()
            os.remove(Path(output_file_name))