





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase__           =			logging.get_logger(__name__)

UpperCamelCase__           =			{"vocab_file": "spiece.model"}

UpperCamelCase__           =			{
    "vocab_file": {
        "TsinghuaAI/CPM-Generate": "https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model",
    }
}

class 							__SCREAMING_SNAKE_CASE       (  _a							):






							def __init__(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=False	,			__lowerCAmelCase=True	,			__lowerCAmelCase=False	,			__lowerCAmelCase="<s>"	,			__lowerCAmelCase="</s>"	,			__lowerCAmelCase="<unk>"	,			__lowerCAmelCase="<sep>"	,			__lowerCAmelCase="<pad>"	,			__lowerCAmelCase="<cls>"	,			__lowerCAmelCase="<mask>"	,			__lowerCAmelCase=["<eop>", "<eod>"]	,			__lowerCAmelCase = None	,			**__lowerCAmelCase	,			):
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else mask_token

														UpperCamelCase__					  =      {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=__lowerCAmelCase	,			remove_space=__lowerCAmelCase	,			keep_accents=__lowerCAmelCase	,			bos_token=__lowerCAmelCase	,			eos_token=__lowerCAmelCase	,			unk_token=__lowerCAmelCase	,			sep_token=__lowerCAmelCase	,			pad_token=__lowerCAmelCase	,			cls_token=__lowerCAmelCase	,			mask_token=__lowerCAmelCase	,			additional_special_tokens=__lowerCAmelCase	,			sp_model_kwargs=self.sp_model_kwargs	,			**__lowerCAmelCase	,			)

														UpperCamelCase__					  =      3

														UpperCamelCase__					  =      do_lower_case
														UpperCamelCase__					  =      remove_space
														UpperCamelCase__					  =      keep_accents
														UpperCamelCase__					  =      vocab_file

														UpperCamelCase__					  =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(__lowerCAmelCase	)

														try:
																					import jieba
														except ModuleNotFoundError as error:
																					raise error.__class__(
																					    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
																					    """See https://pypi.org/project/jieba/ for installation."""	)
														UpperCamelCase__					  =      jieba
														UpperCamelCase__					  =      str.maketrans(""" \n"""	,			"""\u2582\u2583"""	)






							@property
							# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
							def 					_lowerCamelCase		(    self	):
														return len(self.sp_model	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      {self.convert_ids_to_tokens(__lowerCAmelCase	): i for i in range(self.vocab_size	)}
														vocab.update(self.added_tokens_encoder	)
														return vocab






							def __getstate__(    self	):
														UpperCamelCase__					  =      self.__dict__.copy()
														UpperCamelCase__					  =      None
														return state






							def __setstate__(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      d

														# for backward compatibility
														if not hasattr(self	,			"""sp_model_kwargs"""	):
																					UpperCamelCase__					  =      {}

														UpperCamelCase__					  =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(self.vocab_file	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														if self.remove_space:
																					UpperCamelCase__					  =      """ """.join(inputs.strip().split()	)
														else:
																					UpperCamelCase__					  =      inputs
														UpperCamelCase__					  =      outputs.replace("""``"""	,			"""\""""	).replace("""''"""	,			"""\""""	)

														if not self.keep_accents:
																					UpperCamelCase__					  =      unicodedata.normalize("""NFKD"""	,			__lowerCAmelCase	)
																					UpperCamelCase__					  =      """""".join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase	)]	)
														if self.do_lower_case:
																					UpperCamelCase__					  =      outputs.lower()

														return outputs






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      self.preprocess_text(__lowerCAmelCase	)
														UpperCamelCase__					  =      self.sp_model.encode(__lowerCAmelCase	,			out_type=__lowerCAmelCase	)
														UpperCamelCase__					  =      []
														for piece in pieces:
																					if len(__lowerCAmelCase	) > 1 and piece[-1] == str(""","""	) and piece[-2].isdigit():
																												UpperCamelCase__					  =      self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase	,			""""""	)	)
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]	) == 1:
																																										UpperCamelCase__					  =      cur_pieces[1:]
																																			else:
																																										UpperCamelCase__					  =      cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]	)
																												new_pieces.extend(__lowerCAmelCase	)
																					else:
																												new_pieces.append(__lowerCAmelCase	)

														return new_pieces






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.sp_model.PieceToId(__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.sp_model.IdToPiece(__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      """""".join(__lowerCAmelCase	).replace(__lowerCAmelCase	,			""" """	).strip()
														return out_string






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      [self.sep_token_id]
														UpperCamelCase__					  =      [self.cls_token_id]
														if token_ids_a is None:
																					return token_ids_a + sep + cls
														return token_ids_a + sep + token_ids_a + sep + cls






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	,			__lowerCAmelCase = False	):

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__lowerCAmelCase	,			token_ids_a=__lowerCAmelCase	,			already_has_special_tokens=__lowerCAmelCase	)

														if token_ids_a is not None:
																					return ([0] * len(__lowerCAmelCase	)) + [1] + ([0] * len(__lowerCAmelCase	)) + [1, 1]
														return ([0] * len(__lowerCAmelCase	)) + [1, 1]






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      [self.sep_token_id]
														UpperCamelCase__					  =      [2]

														if token_ids_a is None:
																					return len(token_ids_a + sep	) * [0] + cls_segment_id
														return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														if not os.path.isdir(__lowerCAmelCase	):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
																					return
														UpperCamelCase__					  =      os.path.join(
														    __lowerCAmelCase	,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

														if os.path.abspath(self.vocab_file	) != os.path.abspath(__lowerCAmelCase	) and os.path.isfile(self.vocab_file	):
																					copyfile(self.vocab_file	,			__lowerCAmelCase	)
														elif not os.path.isfile(self.vocab_file	):
																					with open(__lowerCAmelCase	,			"""wb"""	) as fi:
																												UpperCamelCase__					  =      self.sp_model.serialized_model_proto()
																												fi.write(__lowerCAmelCase	)

														return (out_vocab_file,)







							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														UpperCamelCase__					  =      super()._decode(*__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      text.replace(""" """	,			""""""	).replace("""\u2582"""	,			""" """	).replace("""\u2583"""	,			"""\n"""	)
														return text











from manim import *

class 							__SCREAMING_SNAKE_CASE       (  _a							):






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      Rectangle(height=0.5	,			width=0.5	)
														UpperCamelCase__					  =      Rectangle(height=0.25	,			width=0.25	)
														UpperCamelCase__					  =      Rectangle(height=0.46	,			width=0.46	).set_stroke(width=0	)

														UpperCamelCase__					  =      [mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      [mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      VGroup(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      Text("""CPU"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														cpu.move_to([-2.5, -0.5, 0]	)
														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      [mem.copy() for i in range(4	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      Text("""GPU"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														gpu.move_to([-1, -1, 0]	)
														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      [mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)

														UpperCamelCase__					  =      Text("""Model"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														model.move_to([3, -1.0, 0]	)
														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      []
														UpperCamelCase__					  =      []
														UpperCamelCase__					  =      []

														for i, rect in enumerate(__lowerCAmelCase	):
																					rect.set_stroke(__lowerCAmelCase	)

																					UpperCamelCase__					  =      Rectangle(height=0.46 / 4	,			width=0.46 / 3	).set_stroke(width=0.0	).set_fill(__lowerCAmelCase	,			opacity=0.7	)

																					if i == 0:
																												cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT	)	,			buff=0.02	,			direction=__lowerCAmelCase	)
																												cpu_target.set_x(cpu_target.get_x() + 0.1	)
																					elif i == 3:
																												cpu_target.next_to(model_cpu_arr[0]	,			direction=__lowerCAmelCase	,			buff=0.0	)
																					else:
																												cpu_target.next_to(model_cpu_arr[i - 1]	,			direction=__lowerCAmelCase	,			buff=0.0	)
																					self.add(__lowerCAmelCase	)
																					model_cpu_arr.append(__lowerCAmelCase	)

														self.add(*__lowerCAmelCase	,			*__lowerCAmelCase	,			*__lowerCAmelCase	)

														UpperCamelCase__					  =      [mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)

														UpperCamelCase__					  =      Text("""Loaded Checkpoint"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														checkpoint.move_to([3, 0.5, 0]	)

														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      []
														UpperCamelCase__					  =      []

														for i, rect in enumerate(__lowerCAmelCase	):
																					UpperCamelCase__					  =      fill.copy().set_fill(__lowerCAmelCase	,			opacity=0.7	)
																					target.move_to(__lowerCAmelCase	)
																					ckpt_arr.append(__lowerCAmelCase	)

																					UpperCamelCase__					  =      target.copy()
																					if i < 5:
																												cpu_target.move_to(cpu_left_col_base[i + 1]	)
																					else:
																												cpu_target.move_to(cpu_right_col_base[i - 5]	)
																					ckpt_cpu_arr.append(__lowerCAmelCase	)
														self.add(*__lowerCAmelCase	,			*__lowerCAmelCase	)

														UpperCamelCase__					  =      Square(side_length=2.2	)
														key.move_to([-5, 2, 0]	)

														UpperCamelCase__					  =      MarkupText(
														    f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""	,			font_size=18	,			)

														key_text.move_to([-5, 2.4, 0]	)

														self.add(__lowerCAmelCase	,			__lowerCAmelCase	)

														UpperCamelCase__					  =      MarkupText(
														    f"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""	,			font_size=18	,			)

														blue_text.next_to(__lowerCAmelCase	,			DOWN * 2.4	,			aligned_edge=key_text.get_left()	)
														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      MarkupText(
														    f"""Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device."""	,			font_size=24	,			)
														step_a.move_to([2, 2, 0]	)

														UpperCamelCase__					  =      [meta_mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      [meta_mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      VGroup(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      Text("""Disk"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														disk.move_to([-4.0, -1.25, 0]	)
														self.play(Write(__lowerCAmelCase	,			run_time=3	)	,			Write(__lowerCAmelCase	,			run_time=1	)	,			Create(__lowerCAmelCase	,			run_time=1	)	)

														UpperCamelCase__					  =      []
														for i, rect in enumerate(__lowerCAmelCase	):
																					UpperCamelCase__					  =      rect.copy()
																					target.generate_target()
																					target.target.move_to(disk_left_col_base[i]	).scale(0.5	)
																					animations.append(MoveToTarget(__lowerCAmelCase	,			run_time=1.5	)	)
														self.play(*__lowerCAmelCase	)

														self.play(FadeOut(__lowerCAmelCase	)	)

														UpperCamelCase__					  =      MarkupText(f"""Then, the checkpoint is removed from memory\nthrough garbage collection."""	,			font_size=24	)
														step_a.move_to([2, 2, 0]	)

														self.play(Write(__lowerCAmelCase	,			run_time=3	)	)

														self.play(
														    FadeOut(__lowerCAmelCase	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			*__lowerCAmelCase	)	,			)

														self.wait()




