




"""simple docstring"""







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_	     =       logging.get_logger(__name__)
SCREAMING_SNAKE_CASE_	     =       {'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE_	     =       {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

SCREAMING_SNAKE_CASE_	     =       {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

SCREAMING_SNAKE_CASE_	     =       '▁'

class 				snake_case_			(					lowerCamelCase_			):







   """simple docstring"""




   A_				      =       VOCAB_FILES_NAMES
   A_				      =       PRETRAINED_VOCAB_FILES_MAP
   A_				      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







   def __init__(		self       ,					lowerCamelCase_       ,					lowerCamelCase_=True       ,					lowerCamelCase_=True       ,					lowerCamelCase_=False       ,					lowerCamelCase_="[CLS]"       ,					lowerCamelCase_="[SEP]"       ,					lowerCamelCase_="<unk>"       ,					lowerCamelCase_="[SEP]"       ,					lowerCamelCase_="<pad>"       ,					lowerCamelCase_="[CLS]"       ,					lowerCamelCase_="[MASK]"       ,					lowerCamelCase_ = None       ,					**lowerCamelCase_       ,					)				->   None:
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.
       UpperCamelCase              =	(
           AddedToken(lowerCamelCase_       ,					lstrip=lowerCamelCase_       ,					rstrip=lowerCamelCase_       ,					normalized=lowerCamelCase_)
           if isinstance(lowerCamelCase_       ,					lowerCamelCase_)
           else mask_token
       )

       UpperCamelCase              =	{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=lowerCamelCase_       ,					remove_space=lowerCamelCase_       ,					keep_accents=lowerCamelCase_       ,					bos_token=lowerCamelCase_       ,					eos_token=lowerCamelCase_       ,					unk_token=lowerCamelCase_       ,					sep_token=lowerCamelCase_       ,					pad_token=lowerCamelCase_       ,					cls_token=lowerCamelCase_       ,					mask_token=lowerCamelCase_       ,					sp_model_kwargs=self.sp_model_kwargs       ,					**lowerCamelCase_       ,					)

       UpperCamelCase              =	do_lower_case
       UpperCamelCase              =	remove_space
       UpperCamelCase              =	keep_accents
       UpperCamelCase              =	vocab_file

       UpperCamelCase              =	spm.SentencePieceProcessor(**self.sp_model_kwargs)
       self.sp_model.Load(lowerCamelCase_)







   @property
   def   UpperCAmelCase__			(		self)				->   Union[str, Any]:
       return len(self.sp_model)







   def   UpperCAmelCase__			(		self)				->   Any:
       UpperCamelCase              =	{self.convert_ids_to_tokens(lowerCamelCase_): i for i in range(self.vocab_size)}
       vocab.update(self.added_tokens_encoder)
       return vocab







   def __getstate__(		self)				->   Union[str, Any]:
       UpperCamelCase              =	self.__dict__.copy()
       UpperCamelCase              =	None
       return state







   def __setstate__(		self       ,					lowerCamelCase_)				->   Optional[Any]:
       UpperCamelCase              =	d

       # for backward compatibility
       if not hasattr(self       ,					'''sp_model_kwargs'''):
           UpperCamelCase              =	{}

       UpperCamelCase              =	spm.SentencePieceProcessor(**self.sp_model_kwargs)
       self.sp_model.Load(self.vocab_file)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   List[str]:
       if self.remove_space:
           UpperCamelCase              =	''' '''.join(inputs.strip().split())
       else:
           UpperCamelCase              =	inputs
       UpperCamelCase              =	outputs.replace('''``'''       ,					'''"''').replace('''\'\''''       ,					'''"''')

       if not self.keep_accents:
           UpperCamelCase              =	unicodedata.normalize('''NFKD'''       ,					lowerCamelCase_)
           UpperCamelCase              =	''''''.join([c for c in outputs if not unicodedata.combining(lowerCamelCase_)])
       if self.do_lower_case:
           UpperCamelCase              =	outputs.lower()

       return outputs







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   List[str]:
       UpperCamelCase              =	self.preprocess_text(lowerCamelCase_)
       UpperCamelCase              =	self.sp_model.encode(lowerCamelCase_       ,					out_type=lowerCamelCase_)
       UpperCamelCase              =	[]
       for piece in pieces:
           if len(lowerCamelCase_) > 1 and piece[-1] == str(''',''') and piece[-2].isdigit():
               UpperCamelCase              =	self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase_       ,					''''''))
               if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                   if len(cur_pieces[0]) == 1:
                       UpperCamelCase              =	cur_pieces[1:]
                   else:
                       UpperCamelCase              =	cur_pieces[0][1:]
               cur_pieces.append(piece[-1])
               new_pieces.extend(lowerCamelCase_)
           else:
               new_pieces.append(lowerCamelCase_)

       return new_pieces







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Union[str, Any]:
       return self.sp_model.PieceToId(lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Any:
       return self.sp_model.IdToPiece(lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   str:
       UpperCamelCase              =	[]
       UpperCamelCase              =	''''''
       UpperCamelCase              =	False
       for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
               if not prev_is_special:
                   out_string += " "
               out_string += self.sp_model.decode(lowerCamelCase_) + token
               UpperCamelCase              =	True
               UpperCamelCase              =	[]
           else:
               current_sub_tokens.append(lowerCamelCase_)
               UpperCamelCase              =	False
       out_string += self.sp_model.decode(lowerCamelCase_)
       return out_string.strip()







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   List[int]:
       UpperCamelCase              =	[self.sep_token_id]
       UpperCamelCase              =	[self.cls_token_id]
       if token_ids_a is None:
           return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None       ,					lowerCamelCase_ = False)				->   List[int]:

       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=lowerCamelCase_       ,					token_ids_a=lowerCamelCase_       ,					already_has_special_tokens=lowerCamelCase_)

       if token_ids_a is not None:
           return [1] + ([0] * len(lowerCamelCase_)) + [1] + ([0] * len(lowerCamelCase_)) + [1]
       return [1] + ([0] * len(lowerCamelCase_)) + [1]







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   List[int]:
       UpperCamelCase              =	[self.sep_token_id]
       UpperCamelCase              =	[self.cls_token_id]

       if token_ids_a is None:
           return len(cls + token_ids_a + sep) * [0]
       return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   Tuple[str]:
       if not os.path.isdir(lowerCamelCase_):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory')
           return
       UpperCamelCase              =	os.path.join(
           lowerCamelCase_       ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

       if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCamelCase_) and os.path.isfile(self.vocab_file):
           copyfile(self.vocab_file       ,					lowerCamelCase_)
       elif not os.path.isfile(self.vocab_file):
           with open(lowerCamelCase_       ,					'''wb''') as fi:
               UpperCamelCase              =	self.sp_model.serialized_model_proto()
               fi.write(lowerCamelCase_)

       return (out_vocab_file,)




from manim import *



class      _UpperCAmelCase      (							A__  ):




     def 		snake_case_		(					self):
         A__					=       Rectangle(height=0.5			,    width=0.5)
         A__					=       Rectangle(height=0.2_5			,    width=0.2_5)
         A__					=       Rectangle(height=0.4_6			,    width=0.4_6).set_stroke(width=0)

         A__					=       [mem.copy() for i in range(6)]
         A__					=       [mem.copy() for i in range(6)]
         A__					=       VGroup(*a__).arrange(a__			,    buff=0)
         A__					=       VGroup(*a__).arrange(a__			,    buff=0)
         A__					=       VGroup(a__			,    a__).arrange(a__			,    buff=0)
         A__					=       Text('''CPU'''			,    font_size=2_4)
         A__					=       Group(a__			,    a__).arrange(a__			,    buff=0.5			,    aligned_edge=a__)
         cpu.move_to([-2.5, -0.5, 0])
         self.add(a__)

         A__					=       [mem.copy() for i in range(4)]
         A__					=       VGroup(*a__).arrange(a__			,    buff=0)
         A__					=       Text('''GPU'''			,    font_size=2_4)
         A__					=       Group(a__			,    a__).arrange(a__			,    buff=0.5			,    aligned_edge=a__)
         gpu.move_to([-1, -1, 0])
         self.add(a__)

         A__					=       [mem.copy() for i in range(6)]
         A__					=       VGroup(*a__).arrange(a__			,    buff=0)

         A__					=       Text('''Model'''			,    font_size=2_4)
         A__					=       Group(a__			,    a__).arrange(a__			,    buff=0.5			,    aligned_edge=a__)
         model.move_to([3, -1.0, 0])
         self.add(a__)

         A__					=       []
         A__					=       []
         A__					=       []

         for i, rect in enumerate(a__):
             rect.set_stroke(a__)

             A__					=       Rectangle(height=0.4_6 / 4			,    width=0.4_6 / 3).set_stroke(width=0.0).set_fill(a__			,    opacity=0.7)

             if i == 0:
                 cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT)			,    buff=0.0_2			,    direction=a__)
                 cpu_target.set_x(cpu_target.get_x() + 0.1)
             elif i == 3:
                 cpu_target.next_to(model_cpu_arr[0]			,    direction=a__			,    buff=0.0)
             else:
                 cpu_target.next_to(model_cpu_arr[i - 1]			,    direction=a__			,    buff=0.0)
             self.add(a__)
             model_cpu_arr.append(a__)

         self.add(*a__			,    *a__			,    *a__)

         A__					=       [mem.copy() for i in range(6)]
         A__					=       VGroup(*a__).arrange(a__			,    buff=0)

         A__					=       Text('''Loaded Checkpoint'''			,    font_size=2_4)
         A__					=       Group(a__			,    a__).arrange(a__			,    buff=0.5			,    aligned_edge=a__)
         checkpoint.move_to([3, 0.5, 0])

         self.add(a__)

         A__					=       []
         A__					=       []

         for i, rect in enumerate(a__):
             A__					=       fill.copy().set_fill(a__			,    opacity=0.7)
             target.move_to(a__)
             ckpt_arr.append(a__)

             A__					=       target.copy()
             if i < 5:
                 cpu_target.move_to(cpu_left_col_base[i + 1])
             else:
                 cpu_target.move_to(cpu_right_col_base[i - 5])
             ckpt_cpu_arr.append(a__)
         self.add(*a__			,    *a__)

         A__					=       Square(side_length=2.2)
         key.move_to([-5, 2, 0])

         A__					=       MarkupText(
             F"<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"			,    font_size=1_8			,    )

         key_text.move_to([-5, 2.4, 0])

         self.add(a__			,    a__)

         A__					=       MarkupText(
             F"<span fgcolor='{BLUE}'>●</span> Checkpoint"			,    font_size=1_8			,    )

         blue_text.next_to(a__			,    DOWN * 2.4			,    aligned_edge=key_text.get_left())
         self.add(a__)

         A__					=       MarkupText(
             F"Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device."			,    font_size=2_4			,    )
         step_a.move_to([2, 2, 0])

         A__					=       [meta_mem.copy() for i in range(6)]
         A__					=       [meta_mem.copy() for i in range(6)]
         A__					=       VGroup(*a__).arrange(a__			,    buff=0)
         A__					=       VGroup(*a__).arrange(a__			,    buff=0)
         A__					=       VGroup(a__			,    a__).arrange(a__			,    buff=0)
         A__					=       Text('''Disk'''			,    font_size=2_4)
         A__					=       Group(a__			,    a__).arrange(a__			,    buff=0.5			,    aligned_edge=a__)
         disk.move_to([-4.0, -1.2_5, 0])
         self.play(Write(a__			,    run_time=3)			,    Write(a__			,    run_time=1)			,    Create(a__			,    run_time=1))

         A__					=       []
         for i, rect in enumerate(a__):
             A__					=       rect.copy()
             target.generate_target()
             target.target.move_to(disk_left_col_base[i]).scale(0.5)
             animations.append(MoveToTarget(a__			,    run_time=1.5))
         self.play(*a__)

         self.play(FadeOut(a__))

         A__					=       MarkupText(F"Then, the checkpoint is removed from memory\nthrough garbage collection."			,    font_size=2_4)
         step_a.move_to([2, 2, 0])

         self.play(Write(a__			,    run_time=3))

         self.play(
             FadeOut(a__			,    a__			,    *a__			,    *a__)			,    )

         self.wait()
