




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A        =    logging.get_logger(__name__)

__A        =    "▁"

__A        =    {"vocab_file": "sentencepiece.bpe.model"}

__A        =    {
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

__A        =    {
    "xlm-roberta-base": 5_12,
    "xlm-roberta-large": 5_12,
    "xlm-roberta-large-finetuned-conll02-dutch": 5_12,
    "xlm-roberta-large-finetuned-conll02-spanish": 5_12,
    "xlm-roberta-large-finetuned-conll03-english": 5_12,
    "xlm-roberta-large-finetuned-conll03-german": 5_12,
}







class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			snake_case_   		=       VOCAB_FILES_NAMES
			snake_case_   		=       PRETRAINED_VOCAB_FILES_MAP
			snake_case_   		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case_   		=       ['''input_ids''', '''attention_mask''']


			def __init__(      self			,						lowerCamelCase__			,						lowerCamelCase__="<s>"			,						lowerCamelCase__="</s>"			,						lowerCamelCase__="</s>"			,						lowerCamelCase__="<s>"			,						lowerCamelCase__="<unk>"			,						lowerCamelCase__="<pad>"			,						lowerCamelCase__="<mask>"			,						lowerCamelCase__ = None			,						**lowerCamelCase__			,						)    ->      None:



							'''simple docstring'''
							# Mask token behave like a normal word, i.e. include the space before it
							__lowerCamelCase		       =    AddedToken(lowerCamelCase__			,						lstrip=lowerCamelCase__			,						rstrip=lowerCamelCase__							) if isinstance(lowerCamelCase__			,						lowerCamelCase__							) else mask_token

							__lowerCamelCase		       =    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=lowerCamelCase__			,						eos_token=lowerCamelCase__			,						unk_token=lowerCamelCase__			,						sep_token=lowerCamelCase__			,						cls_token=lowerCamelCase__			,						pad_token=lowerCamelCase__			,						mask_token=lowerCamelCase__			,						sp_model_kwargs=self.sp_model_kwargs			,						**lowerCamelCase__			,						)

							__lowerCamelCase		       =    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(str(lowerCamelCase__							)							)
							__lowerCamelCase		       =    vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							__lowerCamelCase		       =    {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							__lowerCamelCase		       =    1

							__lowerCamelCase		       =    len(self.sp_model							) + self.fairseq_offset
							__lowerCamelCase		       =    {v: k for k, v in self.fairseq_tokens_to_ids.items()}


			def __getstate__(      self							)    ->      Dict:



							'''simple docstring'''
							__lowerCamelCase		       =    self.__dict__.copy()
							__lowerCamelCase		       =    None
							__lowerCamelCase		       =    self.sp_model.serialized_model_proto()
							return state


			def __setstate__(      self			,						lowerCamelCase__							)    ->      Dict:



							'''simple docstring'''
							__lowerCamelCase		       =    d

							# for backward compatibility
							if not hasattr(self			,						'sp_model_kwargs'							):
											__lowerCamelCase		       =    {}

							__lowerCamelCase		       =    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto							)


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      List[int]:



							'''simple docstring'''

							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							__lowerCamelCase		       =    [self.cls_token_id]
							__lowerCamelCase		       =    [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None			,						lowerCamelCase__ = False							)    ->      List[int]:



							'''simple docstring'''

							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCamelCase__			,						token_ids_a=lowerCamelCase__			,						already_has_special_tokens=lowerCamelCase__							)

							if token_ids_a is None:
											return [1] + ([0] * len(lowerCamelCase__							)) + [1]
							return [1] + ([0] * len(lowerCamelCase__							)) + [1, 1] + ([0] * len(lowerCamelCase__							)) + [1]


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      List[int]:



							'''simple docstring'''
							__lowerCamelCase		       =    [self.sep_token_id]
							__lowerCamelCase		       =    [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]


			@property
			def       lowercase_    (      self							)    ->      Optional[Any]:



							'''simple docstring'''
							return len(self.sp_model							) + self.fairseq_offset + 1  # Add the <mask> token


			def       lowercase_    (      self							)    ->      Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    {self.convert_ids_to_tokens(lowerCamelCase__							): i for i in range(self.vocab_size							)}
							vocab.update(self.added_tokens_encoder							)
							return vocab


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      List[str]:



							'''simple docstring'''
							return self.sp_model.encode(lowerCamelCase__			,						out_type=lowerCamelCase__							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Optional[Any]:



							'''simple docstring'''
							if token in self.fairseq_tokens_to_ids:
											return self.fairseq_tokens_to_ids[token]
							__lowerCamelCase		       =    self.sp_model.PieceToId(lowerCamelCase__							)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      List[Any]:



							'''simple docstring'''
							if index in self.fairseq_ids_to_tokens:
											return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    ''.join(lowerCamelCase__							).replace(lowerCamelCase__			,						' '							).strip()
							return out_string






			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      Tuple[str]:



							'''simple docstring'''
							if not os.path.isdir(lowerCamelCase__							):
											logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
											return
							__lowerCamelCase		       =    os.path.join(
							    lowerCamelCase__			,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCamelCase__							) and os.path.isfile(self.vocab_file							):
											copyfile(self.vocab_file			,						lowerCamelCase__							)
							elif not os.path.isfile(self.vocab_file							):
											with open(lowerCamelCase__			,						'wb'							) as fi:
															__lowerCamelCase		       =    self.sp_model.serialized_model_proto()
															fi.write(lowerCamelCase__							)

							return (out_vocab_file,)




import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
    import torch




@is_pipeline_test
class        UpperCAmelCase_			( unittest.TestCase     ):


  '''simple docstring'''







  a__           =		MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
  a__           =		TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
  def 							_lowercase							( self			: str			,							UpperCamelCase__			: Optional[int]			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       TextaTextGenerationPipeline(model=UpperCamelCase__			,							tokenizer=UpperCamelCase__    )
    return generator, ["Something to write", "Something else"]
  def 							_lowercase							( self			: Any			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: List[str]    )			->	Optional[Any]:







    """simple docstring"""






    __magic_name__				     =       generator("""Something there"""    )
    self.assertEqual(UpperCamelCase__			,							[{"""generated_text""": ANY(UpperCamelCase__    )}]    )
    # These are encoder decoder, they don't just append to incoming string
    self.assertFalse(outputs[0]["""generated_text"""].startswith("""Something there"""    )    )

    __magic_name__				     =       generator(["""This is great !""", """Something else"""]			,							num_return_sequences=2			,							do_sample=UpperCamelCase__    )
    self.assertEqual(
        UpperCamelCase__			,							[
            [{"""generated_text""": ANY(UpperCamelCase__    )}, {"""generated_text""": ANY(UpperCamelCase__    )}],
            [{"""generated_text""": ANY(UpperCamelCase__    )}, {"""generated_text""": ANY(UpperCamelCase__    )}],
        ]			,							)

    __magic_name__				     =       generator(
        ["""This is great !""", """Something else"""]			,							num_return_sequences=2			,							batch_size=2			,							do_sample=UpperCamelCase__    )
    self.assertEqual(
        UpperCamelCase__			,							[
            [{"""generated_text""": ANY(UpperCamelCase__    )}, {"""generated_text""": ANY(UpperCamelCase__    )}],
            [{"""generated_text""": ANY(UpperCamelCase__    )}, {"""generated_text""": ANY(UpperCamelCase__    )}],
        ]			,							)

    with self.assertRaises(UpperCamelCase__    ):
      generator(4    )
  @require_torch
  def 							_lowercase							( self			: Optional[int]    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       pipeline("""text2text-generation"""			,							model="""patrickvonplaten/t5-tiny-random"""			,							framework="""pt"""    )
    # do_sample=False necessary for reproducibility
    __magic_name__				     =       generator("""Something there"""			,							do_sample=UpperCamelCase__    )
    self.assertEqual(UpperCamelCase__			,							[{"""generated_text""": """"""}]    )

    __magic_name__				     =       3
    __magic_name__				     =       generator(
        """Something there"""			,							num_return_sequences=UpperCamelCase__			,							num_beams=UpperCamelCase__			,							)
    __magic_name__				     =       [
        {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide Beide"""},
        {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide"""},
        {"""generated_text""": """"""},
    ]
    self.assertEqual(UpperCamelCase__			,							UpperCamelCase__    )

    __magic_name__				     =       generator("""This is a test"""			,							do_sample=UpperCamelCase__			,							num_return_sequences=2			,							return_tensors=UpperCamelCase__    )
    self.assertEqual(
        UpperCamelCase__			,							[
            {"""generated_token_ids""": ANY(torch.Tensor    )},
            {"""generated_token_ids""": ANY(torch.Tensor    )},
        ]			,							)
    __magic_name__				     =       generator.model.config.eos_token_id
    __magic_name__				     =       """<pad>"""
    __magic_name__				     =       generator(
        ["""This is a test""", """This is a second test"""]			,							do_sample=UpperCamelCase__			,							num_return_sequences=2			,							batch_size=2			,							return_tensors=UpperCamelCase__			,							)
    self.assertEqual(
        UpperCamelCase__			,							[
            [
                {"""generated_token_ids""": ANY(torch.Tensor    )},
                {"""generated_token_ids""": ANY(torch.Tensor    )},
            ],
            [
                {"""generated_token_ids""": ANY(torch.Tensor    )},
                {"""generated_token_ids""": ANY(torch.Tensor    )},
            ],
        ]			,							)





  @require_tf
  def 							_lowercase							( self			: int    )			->	str:







    """simple docstring"""






    __magic_name__				     =       pipeline("""text2text-generation"""			,							model="""patrickvonplaten/t5-tiny-random"""			,							framework="""tf"""    )
    # do_sample=False necessary for reproducibility
    __magic_name__				     =       generator("""Something there"""			,							do_sample=UpperCamelCase__    )
    self.assertEqual(UpperCamelCase__			,							[{"""generated_text""": """"""}]    )

