


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_mbart import MBartTokenizer
else:
 SCREAMING_SNAKE_CASE__					:     Tuple       =     None


SCREAMING_SNAKE_CASE__					:     str       =     logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__					:     List[Any]       =     {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE__					:     Dict       =     {
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE__					:     str       =     {
    'facebook/mbart-large-en-ro': 1_024,
    'facebook/mbart-large-cc25': 1_024,
}

# fmt: off
SCREAMING_SNAKE_CASE__					:     Dict       =     ['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']


class 		_SCREAMING_SNAKE_CASE     (						A     ):
     __SCREAMING_SNAKE_CASE     					=    VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE     					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __SCREAMING_SNAKE_CASE     					=    PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE     					=    ['''input_ids''', '''attention_mask''']
     __SCREAMING_SNAKE_CASE     					=    MBartTokenizer

     __SCREAMING_SNAKE_CASE     					=    []
     __SCREAMING_SNAKE_CASE     					=    []


     def __init__(  self							,					A_=None							,					A_=None							,					A_="<s>"							,					A_="</s>"							,					A_="</s>"							,					A_="<s>"							,					A_="<unk>"							,					A_="<pad>"							,					A_="<mask>"							,					A_=None							,					A_=None							,					A_=None							,					**A_							,					):
       # Mask token behave like a normal word, i.e. include the space before it
       _UpperCAmelCase		:							Dict							=     AddedToken(A_							,					lstrip=A_							,					rstrip=A_    ) if isinstance(A_							,					A_    ) else mask_token

       super().__init__(
           vocab_file=A_							,					tokenizer_file=A_							,					bos_token=A_							,					eos_token=A_							,					sep_token=A_							,					cls_token=A_							,					unk_token=A_							,					pad_token=A_							,					mask_token=A_							,					src_lang=A_							,					tgt_lang=A_							,					additional_special_tokens=A_							,					**A_							,					)

       _UpperCAmelCase		:							Any							=     vocab_file
       _UpperCAmelCase		:							Tuple							=     False if not self.vocab_file else True

       _UpperCAmelCase		:							Any							=     FAIRSEQ_LANGUAGE_CODES.copy()

       if additional_special_tokens is not None:
         # Only add those special tokens if they are not already there.
         _additional_special_tokens.extend(
             [t for t in additional_special_tokens if t not in _additional_special_tokens]    )

       self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}    )
       _UpperCAmelCase		:							Optional[Any]							=     {
           lang_code: self.convert_tokens_to_ids(A_    ) for lang_code in FAIRSEQ_LANGUAGE_CODES
       }

       _UpperCAmelCase		:							Optional[Any]							=     src_lang if src_lang is not None else """en_XX"""
       _UpperCAmelCase		:							List[str]							=     self.convert_tokens_to_ids(self._src_lang    )
       _UpperCAmelCase		:							Union[str, Any]							=     tgt_lang
       self.set_src_lang_special_tokens(self._src_lang    )


     @property
     def        __snake_case(  self    ):
       return self._src_lang


     @src_lang.setter
     def        __snake_case(  self							,					A_    ):
       _UpperCAmelCase		:							Optional[Any]							=     new_src_lang
       self.set_src_lang_special_tokens(self._src_lang    )


     def        __snake_case(  self							,					A_							,					A_ = None    ):
       if token_ids_a is None:
         return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


     def        __snake_case(  self							,					A_							,					A_ = None    ):
       _UpperCAmelCase		:							List[Any]							=     [self.sep_token_id]
       _UpperCAmelCase		:							List[Any]							=     [self.cls_token_id]

       if token_ids_a is None:
         return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]


     def        __snake_case(  self							,					A_							,					A_							,					A_							,					A_							,					**A_    ):
       if src_lang is None or tgt_lang is None:
         raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
       _UpperCAmelCase		:							Dict							=     src_lang
       _UpperCAmelCase		:							Dict							=     self(A_							,					add_special_tokens=A_							,					return_tensors=A_							,					**A_    )
       _UpperCAmelCase		:							Optional[Any]							=     self.convert_tokens_to_ids(A_    )
       _UpperCAmelCase		:							str							=     tgt_lang_id
       return inputs


     def        __snake_case(  self							,					A_							,					A_ = "en_XX"							,					A_ = None							,					A_ = "ro_RO"							,					**A_							,					):
       _UpperCAmelCase		:							int							=     src_lang
       _UpperCAmelCase		:							int							=     tgt_lang
       return super().prepare_seqaseq_batch(A_							,					A_							,					**A_    )


     def        __snake_case(  self    ):
       return self.set_src_lang_special_tokens(self.src_lang    )


     def        __snake_case(  self    ):
       return self.set_tgt_lang_special_tokens(self.tgt_lang    )


     def        __snake_case(  self							,					A_    ):
       _UpperCAmelCase		:							int							=     self.convert_tokens_to_ids(A_    )
       _UpperCAmelCase		:							Union[str, Any]							=     []
       _UpperCAmelCase		:							Any							=     [self.eos_token_id, self.cur_lang_code]

       _UpperCAmelCase		:							Union[str, Any]							=     self.convert_ids_to_tokens(self.prefix_tokens    )
       _UpperCAmelCase		:							Union[str, Any]							=     self.convert_ids_to_tokens(self.suffix_tokens    )

       _UpperCAmelCase		:							str							=     processors.TemplateProcessing(
           single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str							,					pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str							,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,					self.prefix_tokens + self.suffix_tokens    )    )							,					)


     def        __snake_case(  self							,					A_    ):
       _UpperCAmelCase		:							Optional[int]							=     self.convert_tokens_to_ids(A_    )
       _UpperCAmelCase		:							str							=     []
       _UpperCAmelCase		:							Any							=     [self.eos_token_id, self.cur_lang_code]

       _UpperCAmelCase		:							List[Any]							=     self.convert_ids_to_tokens(self.prefix_tokens    )
       _UpperCAmelCase		:							Union[str, Any]							=     self.convert_ids_to_tokens(self.suffix_tokens    )

       _UpperCAmelCase		:							int							=     processors.TemplateProcessing(
           single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str							,					pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str							,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,					self.prefix_tokens + self.suffix_tokens    )    )							,					)




     def        __snake_case(  self							,					A_							,					A_ = None    ):
       if not self.can_save_slow_tokenizer:
         raise ValueError(
             """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
             """tokenizer."""    )

       if not os.path.isdir(A_    ):
         logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''    )
         return
       _UpperCAmelCase		:							int							=     os.path.join(
           A_							,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(A_    ):
         copyfile(self.vocab_file							,					A_    )

       return (out_vocab_file,)









import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import ClassLabel, Features, Image
from .base import TaskTemplate


@dataclass(frozen=A     )
class 		_SCREAMING_SNAKE_CASE     (						A     ):
     __SCREAMING_SNAKE_CASE     					=    field(default='''image-classification'''							,	metadata={'''include_in_asdict_even_if_is_default''': True}     )
     __SCREAMING_SNAKE_CASE     					=    Features({'''image''': Image()}     )
     __SCREAMING_SNAKE_CASE     					=    Features({'''labels''': ClassLabel}     )
     __SCREAMING_SNAKE_CASE     					=    "image"
     __SCREAMING_SNAKE_CASE     					=    "labels"


     def        __snake_case(  self							,					A_    ):
       if self.label_column not in features:
         raise ValueError(F'''Column {self.label_column} is not present in features.'''    )
       if not isinstance(features[self.label_column]							,					A_    ):
         raise ValueError(F'''Column {self.label_column} is not a ClassLabel.'''    )
       _UpperCAmelCase		:							Tuple							=     copy.deepcopy(self    )
       _UpperCAmelCase		:							str							=     self.label_schema.copy()
       _UpperCAmelCase		:							Optional[Any]							=     features[self.label_column]
       _UpperCAmelCase		:							int							=     label_schema
       return task_template




     @property
     def        __snake_case(  self    ):
       return {
           self.image_column: "image",
           self.label_column: "labels",
       }





