usrbinenv python 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license usage gencardfacebookwmt19 py blue scores as follows pair fairseq transformers fsmt model description intended uses limitations how to use limitations and bias training data eval results data sources bibtex entry and citation info todo os makedirsmodelcarddir existoktrue path os path joinmodelcarddir readme md printfgenerating path with openpath w encodingutf8 as f f writereadme make sure we are under the root of the project repodir pathfile resolve parent parent parent modelcardsdir repodir modelcards for modelname in wmt19ruen wmt19enru wmt19ende wmt19deen base srclang tgtlang modelname split modelcarddir modelcardsdir facebook modelname writemodelcardmodelcarddir srclangsrclang tgtlangtgtlang usr bin env python 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license usage gen card facebook wmt19 py blue scores as follows pair fairseq transformers language src_lang tgt_lang thumbnail tags translation wmt19 facebook license apache 2 0 datasets wmt19 metrics bleu fsmt model description this is a ported version of fairseq wmt19 transformer https github com pytorch fairseq blob master examples wmt19 readme md for src_lang tgt_lang for more details please see facebook fair s wmt19 news translation task submission https arxiv org abs 1907 06616 the abbreviation fsmt stands for fairseqmachinetranslation all four models are available wmt19 en ru https huggingface co facebook wmt19 en ru wmt19 ru en https huggingface co facebook wmt19 ru en wmt19 en de https huggingface co facebook wmt19 en de wmt19 de en https huggingface co facebook wmt19 de en intended uses limitations how to use python from transformers import fsmtforconditionalgeneration fsmttokenizer mname facebook wmt19 src_lang tgt_lang tokenizer fsmttokenizer from_pretrained mname model fsmtforconditionalgeneration from_pretrained mname input texts src_lang input_ids tokenizer encode input return_tensors pt outputs model generate input_ids decoded tokenizer decode outputs 0 skip_special_tokens true print decoded texts tgt_lang limitations and bias the original and this ported model doesn t seem to handle well inputs with repeated sub phrases content gets truncated https discuss huggingface co t issues with translating inputs containing repeated phrases 981 training data pretrained weights were left identical to the original model released by fairseq for more details please see the paper https arxiv org abs 1907 06616 eval results pair fairseq transformers pair scores pair 0 scores pair 1 the score is slightly below the score reported by fairseq since transformers currently doesn t support model ensemble therefore the best performing checkpoint was ported model4 pt re ranking the score was calculated using this code bash git clone https github com huggingface transformers cd transformers export pair pair export data_dir data pair export save_dir data pair export bs 8 export num_beams 15 mkdir p data_dir sacrebleu t wmt19 l pair echo src data_dir val source sacrebleu t wmt19 l pair echo ref data_dir val target echo pair pythonpath src examples seq2seq python examples seq2seq run_eval py facebook wmt19 pair data_dir val source save_dir test_translations txt reference_path data_dir val target score_path save_dir test_bleu json bs bs task translation num_beams num_beams note fairseq reports using a beam of 50 so you should get a slightly higher score if re run with num_beams 50 data sources training etc http www statmt org wmt19 test set http matrix statmt org test_sets newstest2019 tgz 1556572561 bibtex entry and citation info bibtex inproceedings year 2020 title facebook fair s wmt19 news translation task submission ng nathan and yee kyra and baevski alexei and ott myle and auli michael and edunov sergey booktitle proc of wmt todo port model ensemble fairseq uses 4 model checkpoints make sure we are under the root of the project
import os
from pathlib import Path


def write_model_card(model_card_dir, src_lang, tgt_lang):

    texts = {
        "en": "Machine learning is great, isn't it?",
        "ru": "Машинное обучение - это здорово, не так ли?",
        "de": "Maschinelles Lernen ist großartig, oder?",
    }

    
    
    scores = {
        "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "39.20"],
        "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "33.47"],
        "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "42.83"],
        "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "41.35"],
    }
    pair = f"{src_lang}-{tgt_lang}"

    readme = f
    os.makedirs(model_card_dir, exist_ok=True)
    path = os.path.join(model_card_dir, "README.md")
    print(f"Generating {path}")
    with open(path, "w", encoding="utf-8") as f:
        f.write(readme)


repo_dir = Path(__file__).resolve().parent.parent.parent
model_cards_dir = repo_dir / "model_cards"

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
    base, src_lang, tgt_lang = model_name.split("-")
    model_card_dir = model_cards_dir / "facebook" / model_name
    write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)