





# Usage:
# ./gen-card-facebook-wmt19.py

import os
from pathlib import Path


def        SCREAMING_SNAKE_CASE      (       _UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase )			->     List[str]:
    lowerCamelCase__    :      Tuple					 =		{
        'en': 'Machine learning is great, isn\'t it?',
        'ru': 'Машинное обучение - это здорово, не так ли?',
        'de': 'Maschinelles Lernen ist großartig, oder?',
    }

    # BLUE scores as follows:
    # "pair": [fairseq, transformers]
    lowerCamelCase__    :      Tuple					 =		{
        'ru-en': ['[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)', '39.20'],
        'en-ru': ['[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)', '33.47'],
        'en-de': ['[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)', '42.83'],
        'de-en': ['[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)', '41.35'],
    }
    lowerCamelCase__    :      str					 =		F"""{src_lang}-{tgt_lang}"""

    lowerCamelCase__    :      Any					 =		F"""
---
language: 
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- facebook
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}.

For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616).

The abbreviation FSMT stands for FairSeqMachineTranslation

All four models are available:

* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru)
* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en)
* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de)
* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en)

## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = \"facebook/wmt19-{src_lang}-{tgt_lang}\"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = \"{texts[src_lang]}\"
input_ids = tokenizer.encode(input, return_tensors=\"pt\")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias

- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981)

## Training data

Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616).

## Eval results

pair   | fairseq | transformers
-------|---------|----------
{pair}  | {scores[pair][0]} | {scores[pair][1]}

The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support:
- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``).
- re-ranking

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=15
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```
note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`.

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```bibtex
@inproceedings{{...,
  year={{2020}},
  title={{Facebook FAIR's WMT19 News Translation Task Submission}},
  author={{Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}},
  booktitle={{Proc. of WMT}},
}}
```


## TODO

- port model ensemble (fairseq uses 4 model checkpoints)

"""
    os.makedirs(_UpperCAmelCase						,	exist_ok=_UpperCAmelCase )
    lowerCamelCase__    :      Dict					 =		os.path.join(_UpperCAmelCase						,	'README.md' )
    print(F"""Generating {path}""" )
    with open(_UpperCAmelCase						,	'w'						,	encoding='utf-8' ) as f:
        f.write(_UpperCAmelCase )


# make sure we are under the root of the project
_UpperCAmelCase				:       Tuple	   =				Path(__file__).resolve().parent.parent.parent
_UpperCAmelCase				:       List[Any]	   =				repo_dir / """model_cards"""

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
    _UpperCAmelCase   ,_UpperCAmelCase   ,_UpperCAmelCase				:       int	   =				model_name.split("""-""")
    _UpperCAmelCase				:       Tuple	   =				model_cards_dir / """facebook""" / model_name
    write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)









from typing import List, Optional, Union

import torch

from ...models import UNetaDConditionModel, VQModel
from ...pipelines import DiffusionPipeline
from ...pipelines.pipeline_utils import ImagePipelineOutput
from ...schedulers import DDPMScheduler
from ...utils import (
    is_accelerate_available,
    is_accelerate_version,
    logging,
    randn_tensor,
    replace_example_docstring,
)


snake_case				:    Any       				=     logging.get_logger(__name__)  # pylint: disable=invalid-name

snake_case				:    Union[str, Any]       				=     '''
    Examples:
        ```py
        >>> from diffusers import KandinskyV22Pipeline, KandinskyV22PriorPipeline
        >>> import torch

        >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior")
        >>> pipe_prior.to("cuda")
        >>> prompt = "red cat, 4k photo"
        >>> out = pipe_prior(prompt)
        >>> image_emb = out.image_embeds
        >>> zero_image_emb = out.negative_image_embeds
        >>> pipe = KandinskyV22Pipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder")
        >>> pipe.to("cuda")
        >>> image = pipe(
        ...     image_embeds=image_emb,
        ...     negative_image_embeds=zero_image_emb,
        ...     height=768,
        ...     width=768,
        ...     num_inference_steps=50,
        ... ).images
        >>> image[0].save("cat.png")
        ```
'''
def 				__lowerCamelCase						( UpperCAmelCase_						:		List[Any] ,  UpperCAmelCase_						:		Optional[int] ,  UpperCAmelCase_						:		List[str]=8						):




      """simple docstring"""





      a						:List[str] 							=       height // scale_factor**2
      if height % scale_factor**2 != 0:
            new_height += 1
      a						:int 							=       width // scale_factor**2
      if width % scale_factor**2 != 0:
            new_width += 1
      return new_height * scale_factor, new_width * scale_factor







class 		_snake_case  (      _snake_case							):




 def __init__(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  ):
       super().__init__()

       self.register_modules(
           unet=_lowerCamelCase					,  scheduler=_lowerCamelCase					,  movq=_lowerCamelCase					,  )
       a						:Any 							=       2 ** (len(self.movq.config.block_out_channels    ) - 1)




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       if latents is None:
             a						:str 							=       randn_tensor(_lowerCamelCase					,  generator=_lowerCamelCase					,  device=_lowerCamelCase					,  dtype=_lowerCamelCase    )
       else:
             if latents.shape != shape:
                   raise ValueError(F'''Unexpected latents shape, got {latents.shape}, expected {shape}'''    )
             a						:Any 							=       latents.to(_lowerCamelCase    )

       a						:Dict 							=       latents * scheduler.init_noise_sigma
       return latents




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase=0    ):
       if is_accelerate_available():
             from accelerate import cpu_offload
       else:
             raise ImportError('''Please install accelerate via `pip install accelerate`'''    )

       a						:int 							=       torch.device(F'''cuda:{gpu_id}'''    )

       a						:int 							=       [
           self.unet,
           self.movq,
       ]
       for cpu_offloaded_model in models:
             if cpu_offloaded_model is not None:
                   cpu_offload(_lowerCamelCase					,  _lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase=0    ):
       if is_accelerate_available() and is_accelerate_version('''>='''					,  '''0.17.0.dev0'''    ):
             from accelerate import cpu_offload_with_hook
       else:
             raise ImportError('''`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.'''    )

       a						:Any 							=       torch.device(F'''cuda:{gpu_id}'''    )

       if self.device.type != "cpu":
             self.to('''cpu'''					,  silence_dtype_warnings=_lowerCamelCase    )
             torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)

       a						:Tuple 							=       None
       for cpu_offloaded_model in [self.unet, self.movq]:
             a							,				a						:List[str] 							=       cpu_offload_with_hook(_lowerCamelCase					,  _lowerCamelCase					,  prev_module_hook=_lowerCamelCase    )

       # We'll offload the last model manually.
       a						:str 							=       hook




 @property
 # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._execution_device
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       if not hasattr(self.unet					,  '''_hf_hook'''    ):
             return self.device
       for module in self.unet.modules():
             if (
                 hasattr(_lowerCamelCase					,  '''_hf_hook'''    )
                 and hasattr(module._hf_hook					,  '''execution_device'''    )
                 and module._hf_hook.execution_device is not None
             ):
                   return torch.device(module._hf_hook.execution_device    )
       return self.device




 @torch.no_grad()
 @replace_example_docstring(_lowerCamelCase    )
 def __call__(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase = 512					,  _lowerCamelCase = 512					,  _lowerCamelCase = 100					,  _lowerCamelCase = 4.0					,  _lowerCamelCase = 1					,  _lowerCamelCase = None					,  _lowerCamelCase = None					,  _lowerCamelCase = "pil"					,  _lowerCamelCase = True					,  ):
       a						:int 							=       self._execution_device

       a						:Optional[Any] 							=       guidance_scale > 1.0

       if isinstance(_lowerCamelCase					,  _lowerCamelCase    ):
             a						:Union[str, Any] 							=       torch.cat(_lowerCamelCase					,  dim=0    )
       a						:Any 							=       image_embeds.shape[0] * num_images_per_prompt
       if isinstance(_lowerCamelCase					,  _lowerCamelCase    ):
             a						:List[str] 							=       torch.cat(_lowerCamelCase					,  dim=0    )

       if do_classifier_free_guidance:
             a						:Union[str, Any] 							=       image_embeds.repeat_interleave(_lowerCamelCase					,  dim=0    )
             a						:Optional[int] 							=       negative_image_embeds.repeat_interleave(_lowerCamelCase					,  dim=0    )

       a						:Optional[int] 							=       torch.cat([negative_image_embeds, image_embeds]					,  dim=0    ).to(dtype=self.unet.dtype					,  device=_lowerCamelCase    )

       self.scheduler.set_timesteps(_lowerCamelCase					,  device=_lowerCamelCase    )
       a						:Optional[Any] 							=       self.scheduler.timesteps

       a						:List[str] 							=       self.unet.config.in_channels

       a							,				a						:str 							=       downscale_height_and_width(_lowerCamelCase					,  _lowerCamelCase					,  self.movq_scale_factor    )

       # create initial latent
       a						:int 							=       self.prepare_latents(
           (batch_size, num_channels_latents, height, width)					,  image_embeds.dtype					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  self.scheduler					,  )

       for i, t in enumerate(self.progress_bar(_lowerCamelCase    )    ):
             # expand the latents if we are doing classifier free guidance
             a						:Tuple 							=       torch.cat([latents] * 2    ) if do_classifier_free_guidance else latents

             a						:Union[str, Any] 							=       {'''image_embeds''': image_embeds}
             a						:Optional[Any] 							=       self.unet(
                 sample=_lowerCamelCase					,  timestep=_lowerCamelCase					,  encoder_hidden_states=_lowerCamelCase					,  added_cond_kwargs=_lowerCamelCase					,  return_dict=_lowerCamelCase					,  )[0]

             if do_classifier_free_guidance:
                   a							,				a						:Any 							=       noise_pred.split(latents.shape[1]					,  dim=1    )
                   a							,				a						:List[str] 							=       noise_pred.chunk(2    )
                   a							,				a						:int 							=       variance_pred.chunk(2    )
                   a						:List[Any] 							=       noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
                   a						:Optional[int] 							=       torch.cat([noise_pred, variance_pred_text]					,  dim=1    )

             if not (
                 hasattr(self.scheduler.config					,  '''variance_type'''    )
                 and self.scheduler.config.variance_type in ["learned", "learned_range"]
             ):
                   a							,				a						:Tuple 							=       noise_pred.split(latents.shape[1]					,  dim=1    )

             # compute the previous noisy sample x_t -> x_t-1
             a						:int 							=       self.scheduler.step(
                 _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  generator=_lowerCamelCase					,  )[0]
       # post-processing
       a						:int 							=       self.movq.decode(_lowerCamelCase					,  force_not_quantize=_lowerCamelCase    )['''sample''']

       if output_type not in ["pt", "np", "pil"]:
             raise ValueError(F'''Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}'''    )

       if output_type in ["np", "pil"]:
             a						:str 							=       image * 0.5 + 0.5
             a						:List[Any] 							=       image.clamp(0					,  1    )
             a						:str 							=       image.cpu().permute(0					,  2					,  3					,  1    ).float().numpy()

       if output_type == "pil":
             a						:str 							=       self.numpy_to_pil(_lowerCamelCase    )

       if not return_dict:
             return (image,)

       return ImagePipelineOutput(images=_lowerCamelCase    )





