


from manim import *







class 		_snake_case  (      _snake_case							):




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Dict 							=       Rectangle(height=0.5					,  width=0.5    )
       a						:Union[str, Any] 							=       Rectangle(height=0.46					,  width=0.46    ).set_stroke(width=0    )

       a						:List[Any] 							=       [mem.copy() for i in range(6    )]
       a						:int 							=       [mem.copy() for i in range(6    )]
       a						:Optional[Any] 							=       VGroup(*_lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0    )
       a						:List[str] 							=       VGroup(*_lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0    )
       a						:Union[str, Any] 							=       VGroup(_lowerCamelCase					,  _lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0    )
       a						:Any 							=       Text('''CPU'''					,  font_size=24    )
       a						:Dict 							=       Group(_lowerCamelCase					,  _lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0.5					,  aligned_edge=_lowerCamelCase    )
       cpu.move_to([-2.5, -0.5, 0]    )
       self.add(_lowerCamelCase    )

       a						:Optional[Any] 							=       [mem.copy() for i in range(4    )]
       a						:Dict 							=       VGroup(*_lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0    )
       a						:Union[str, Any] 							=       Text('''GPU'''					,  font_size=24    )
       a						:List[str] 							=       Group(_lowerCamelCase					,  _lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0.5					,  aligned_edge=_lowerCamelCase    )
       gpu.move_to([-1, -1, 0]    )
       self.add(_lowerCamelCase    )

       a						:List[str] 							=       [mem.copy() for i in range(6    )]
       a						:Union[str, Any] 							=       VGroup(*_lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0    )

       a						:Optional[int] 							=       Text('''Model'''					,  font_size=24    )
       a						:int 							=       Group(_lowerCamelCase					,  _lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0.5					,  aligned_edge=_lowerCamelCase    )
       model.move_to([3, -1.0, 0]    )
       self.add(_lowerCamelCase    )

       a						:Optional[Any] 							=       []
       for i, rect in enumerate(_lowerCamelCase    ):
             rect.set_stroke(_lowerCamelCase    )
             # target = fill.copy().set_fill(YELLOW, opacity=0.7)
             # target.move_to(rect)
             # self.add(target)

             a						:int 							=       Rectangle(height=0.46 / 4					,  width=0.46 / 3    ).set_stroke(width=0.0    ).set_fill(_lowerCamelCase					,  opacity=0.7    )

             if i == 0:
                   cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT    )					,  buff=0.02					,  direction=_lowerCamelCase    )
                   cpu_target.set_x(cpu_target.get_x() + 0.1    )
             elif i == 3:
                   cpu_target.next_to(cpu_targs[0]					,  direction=_lowerCamelCase					,  buff=0.0    )
             else:
                   cpu_target.next_to(cpu_targs[i - 1]					,  direction=_lowerCamelCase					,  buff=0.0    )
             self.add(_lowerCamelCase    )
             cpu_targs.append(_lowerCamelCase    )

       a						:Optional[Any] 							=       [mem.copy() for i in range(6    )]
       a						:Tuple 							=       VGroup(*_lowerCamelCase    ).arrange(_lowerCamelCase					,  buff=0    )

       a						:Any 							=       Text('''Loaded Checkpoint'''					,  font_size=24    )
       a						:Dict 							=       Group(_lowerCamelCase					,  _lowerCamelCase    ).arrange(_lowerCamelCase					,  aligned_edge=_lowerCamelCase					,  buff=0.4    )
       checkpoint.move_to([3, 0.5, 0]    )

       a						:Optional[int] 							=       Square(side_length=2.2    )
       key.move_to([-5, 2, 0]    )

       a						:Optional[Any] 							=       MarkupText(
           F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''					,  font_size=18					,  )

       key_text.move_to([-5, 2.4, 0]    )

       self.add(_lowerCamelCase					,  _lowerCamelCase    )

       a						:Union[str, Any] 							=       MarkupText(
           F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''					,  font_size=18					,  )

       blue_text.next_to(_lowerCamelCase					,  DOWN * 2.4					,  aligned_edge=key_text.get_left()    )

       a						:Dict 							=       MarkupText(
           F'''Next, a <i><span fgcolor="{BLUE}">second</span></i> model is loaded into memory,\nwith the weights of a <span fgcolor="{BLUE}">single shard</span>.'''					,  font_size=24					,  )
       step_a.move_to([2, 2, 0]    )
       self.play(Write(_lowerCamelCase    )					,  Write(_lowerCamelCase    )    )

       self.play(Write(_lowerCamelCase					,  run_time=1    )					,  Create(_lowerCamelCase					,  run_time=1    )    )

       a						:Optional[Any] 							=       []
       a						:List[Any] 							=       []
       for i, rect in enumerate(_lowerCamelCase    ):
             a						:Tuple 							=       fill.copy().set_fill(_lowerCamelCase					,  opacity=0.7    )
             target.move_to(_lowerCamelCase    )
             first_animations.append(GrowFromCenter(_lowerCamelCase					,  run_time=1    )    )

             a						:Optional[Any] 							=       target.copy()
             cpu_target.generate_target()
             if i < 5:
                   cpu_target.target.move_to(cpu_left_col_base[i + 1]    )
             else:
                   cpu_target.target.move_to(cpu_right_col_base[i - 5]    )
             second_animations.append(MoveToTarget(_lowerCamelCase					,  run_time=1.5    )    )

       self.play(*_lowerCamelCase    )
       self.play(*_lowerCamelCase    )
       self.wait()









import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case				:    Dict       				=     logging.get_logger(__name__)

snake_case				:    Tuple       				=     '''▁'''

snake_case				:    Any       				=     {'''vocab_file''': '''sentencepiece.bpe.model'''}

snake_case				:    Tuple       				=     {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

snake_case				:    int       				=     {
    '''xlm-roberta-base''': 5_12,
    '''xlm-roberta-large''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_12,
}







class 		_snake_case  (      _snake_case							):
 SCREAMING_SNAKE_CASE__      =       VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE__      =       PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE__      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE__      =       ['input_ids', 'attention_mask']




 def __init__(		self					,  _lowerCamelCase					,  _lowerCamelCase="<s>"					,  _lowerCamelCase="</s>"					,  _lowerCamelCase="</s>"					,  _lowerCamelCase="<s>"					,  _lowerCamelCase="<unk>"					,  _lowerCamelCase="<pad>"					,  _lowerCamelCase="<mask>"					,  _lowerCamelCase = None					,  **_lowerCamelCase					,  ):
       # Mask token behave like a normal word, i.e. include the space before it
       a						:Optional[int] 							=       AddedToken(_lowerCamelCase					,  lstrip=_lowerCamelCase					,  rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase					,  _lowerCamelCase    ) else mask_token

       a						:int 							=       {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           bos_token=_lowerCamelCase					,  eos_token=_lowerCamelCase					,  unk_token=_lowerCamelCase					,  sep_token=_lowerCamelCase					,  cls_token=_lowerCamelCase					,  pad_token=_lowerCamelCase					,  mask_token=_lowerCamelCase					,  sp_model_kwargs=self.sp_model_kwargs					,  **_lowerCamelCase					,  )

       a						:Optional[int] 							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(str(_lowerCamelCase    )    )
       a						:str 							=       vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # Mimic fairseq token-to-id alignment for the first 4 token
       a						:Tuple 							=       {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       a						:List[str] 							=       1

       a						:Dict 							=       len(self.sp_model    ) + self.fairseq_offset
       a						:List[Any] 							=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}




 def __getstate__(		self    ):
       a						:List[str] 							=       self.__dict__.copy()
       a						:Optional[int] 							=       None
       a						:int 							=       self.sp_model.serialized_model_proto()
       return state




 def __setstate__(		self					,  _lowerCamelCase    ):
       a						:Union[str, Any] 							=       d

       # for backward compatibility
       if not hasattr(self					,  '''sp_model_kwargs'''    ):
             a						:Union[str, Any] 							=       {}

       a						:Optional[int] 							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):

       if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       a						:List[Any] 							=       [self.cls_token_id]
       a						:Dict 							=       [self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None					,  _lowerCamelCase = False    ):

       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_lowerCamelCase					,  token_ids_a=_lowerCamelCase					,  already_has_special_tokens=_lowerCamelCase    )

       if token_ids_a is None:
             return [1] + ([0] * len(_lowerCamelCase    )) + [1]
       return [1] + ([0] * len(_lowerCamelCase    )) + [1, 1] + ([0] * len(_lowerCamelCase    )) + [1]




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):
       a						:int 							=       [self.sep_token_id]
       a						:int 							=       [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       return len(self.sp_model    ) + self.fairseq_offset + 1  # Add the <mask> token




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Any 							=       {self.convert_ids_to_tokens(_lowerCamelCase    ): i for i in range(self.vocab_size    )}
       vocab.update(self.added_tokens_encoder    )
       return vocab




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       return self.sp_model.encode(_lowerCamelCase					,  out_type=_lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
       a						:Optional[Any] 							=       self.sp_model.PieceToId(_lowerCamelCase    )

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       a						:Tuple 							=       ''''''.join(_lowerCamelCase    ).replace(_lowerCamelCase					,  ''' '''    ).strip()
       return out_string




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):
       if not os.path.isdir(_lowerCamelCase    ):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
             return
       a						:int 							=       os.path.join(
           _lowerCamelCase					,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(_lowerCamelCase    ) and os.path.isfile(self.vocab_file    ):
             copyfile(self.vocab_file					,  _lowerCamelCase    )
       elif not os.path.isfile(self.vocab_file    ):
             with open(_lowerCamelCase					,  '''wb'''    ) as fi:
                   a						:List[Any] 							=       self.sp_model.serialized_model_proto()
                   fi.write(_lowerCamelCase    )

       return (out_vocab_file,)





