


"""simple docstring"""





from manim import *



class 						snake_case_(				a__	):






     def    lowerCamelCase__   (		self     :    Tuple ):
           lowerCAmelCase    :		List[str]				=      Rectangle(height=0.5    , width=0.5 )
           lowerCAmelCase    :		Union[str, Any]				=      Rectangle(height=0.46    , width=0.46 ).set_stroke(width=0 )

           lowerCAmelCase    :		str				=      [mem.copy() for i in range(6 )]
           lowerCAmelCase    :		str				=      [mem.copy() for i in range(6 )]
           lowerCAmelCase    :		List[str]				=      VGroup(*UpperCamelCase_ ).arrange(UpperCamelCase_    , buff=0 )
           lowerCAmelCase    :		Optional[Any]				=      VGroup(*UpperCamelCase_ ).arrange(UpperCamelCase_    , buff=0 )
           lowerCAmelCase    :		Optional[Any]				=      VGroup(UpperCamelCase_    , UpperCamelCase_ ).arrange(UpperCamelCase_    , buff=0 )
           lowerCAmelCase    :		Optional[int]				=      Text('''CPU'''    , font_size=2_4 )
           lowerCAmelCase    :		List[str]				=      Group(UpperCamelCase_    , UpperCamelCase_ ).arrange(UpperCamelCase_    , buff=0.5    , aligned_edge=UpperCamelCase_ )
           cpu.move_to([-2.5, -0.5, 0] )
           self.add(UpperCamelCase_ )

           lowerCAmelCase    :		Optional[Any]				=      [mem.copy() for i in range(1 )]
           lowerCAmelCase    :		Any				=      VGroup(*UpperCamelCase_ ).arrange(UpperCamelCase_    , buff=0 )
           lowerCAmelCase    :		str				=      Text('''GPU'''    , font_size=2_4 )
           lowerCAmelCase    :		int				=      Group(UpperCamelCase_    , UpperCamelCase_ ).arrange(UpperCamelCase_    , buff=0.5    , aligned_edge=UpperCamelCase_ )
           gpu.align_to(UpperCamelCase_    , UpperCamelCase_ )
           gpu.set_x(gpu.get_x() - 1 )

           self.add(UpperCamelCase_ )

           lowerCAmelCase    :		Optional[int]				=      [mem.copy() for i in range(6 )]
           lowerCAmelCase    :		Union[str, Any]				=      VGroup(*UpperCamelCase_ ).arrange(UpperCamelCase_    , buff=0 )

           lowerCAmelCase    :		List[Any]				=      Text('''Model'''    , font_size=2_4 )
           lowerCAmelCase    :		Any				=      Group(UpperCamelCase_    , UpperCamelCase_ ).arrange(UpperCamelCase_    , buff=0.5    , aligned_edge=UpperCamelCase_ )
           model.move_to([3, -1.0, 0] )

           self.play(
               Create(UpperCamelCase_    , run_time=1 )    , Create(UpperCamelCase_    , run_time=1 )    , Create(UpperCamelCase_    , run_time=1 )    , )

           lowerCAmelCase    :		Union[str, Any]				=      MarkupText(
               F'''First, an empty model skeleton is loaded\ninto <span fgcolor=\'{YELLOW}\'>memory</span> without using much RAM.'''    , font_size=2_4    , )

           lowerCAmelCase    :		Optional[int]				=      Square(side_length=2.2 )
           key.move_to([-5, 2, 0] )

           lowerCAmelCase    :		Optional[Any]				=      MarkupText(
               F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''    , font_size=1_8    , )

           key_text.move_to([-5, 2.4, 0] )

           step_a.move_to([2, 2, 0] )
           self.play(Write(UpperCamelCase_    , run_time=2.5 )    , Write(UpperCamelCase_ )    , Write(UpperCamelCase_ ) )

           self.add(UpperCamelCase_ )

           lowerCAmelCase    :		Dict				=      []
           lowerCAmelCase    :		str				=      []
           lowerCAmelCase    :		List[str]				=      []
           for i, rect in enumerate(UpperCamelCase_ ):
                 lowerCAmelCase    :		Optional[int]				=      Rectangle(height=0.46    , width=0.46 ).set_stroke(width=0.0 ).set_fill(UpperCamelCase_    , opacity=0.7 )
                 cpu_target.move_to(UpperCamelCase_ )
                 cpu_target.generate_target()
                 lowerCAmelCase    :		Dict				=      0.46 / 4
                 lowerCAmelCase    :		int				=      0.46 / 3

                 if i == 0:
                       cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT )    , buff=0.02    , direction=UpperCamelCase_ )
                       cpu_target.target.set_x(cpu_target.target.get_x() + 0.1 )
                 elif i == 3:
                       cpu_target.target.next_to(cpu_targs[0].target    , direction=UpperCamelCase_    , buff=0.0 )
                 else:
                       cpu_target.target.next_to(cpu_targs[i - 1].target    , direction=UpperCamelCase_    , buff=0.0 )
                 cpu_targs.append(UpperCamelCase_ )

                 first_animations.append(rect.animate(run_time=0.5 ).set_stroke(UpperCamelCase_ ) )
                 second_animations.append(MoveToTarget(UpperCamelCase_    , run_time=1.5 ) )

           self.play(*UpperCamelCase_ )
           self.play(*UpperCamelCase_ )

           self.wait()







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


A__  :  str								=					logging.get_logger(__name__)

A__  :  Any								=					{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__  :  str								=					{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

A__  :  Union[str, Any]								=					{
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

A__  :  Dict								=					{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}



class 				__snake_case						(     UpperCamelCase_						):
							_a			      = VOCAB_FILES_NAMES
							_a			      = PRETRAINED_VOCAB_FILES_MAP
							_a			      = PRETRAINED_INIT_CONFIGURATION
							_a			      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_a			      = RealmTokenizer




							def __init__(       self     :       int				,				A_     :       Optional[int]=None				,				A_     :       Optional[Any]=None				,				A_     :       Optional[Any]=True				,				A_     :       Optional[int]="[UNK]"				,				A_     :       List[Any]="[SEP]"				,				A_     :       List[Any]="[PAD]"				,				A_     :       Optional[Any]="[CLS]"				,				A_     :       Dict="[MASK]"				,				A_     :       List[Any]=True				,				A_     :       List[str]=None				,				**A_     :       List[str]				,				):
									super().__init__(
									    A_				,				tokenizer_file=A_				,				do_lower_case=A_				,				unk_token=A_				,				sep_token=A_				,				pad_token=A_				,				cls_token=A_				,				mask_token=A_				,				tokenize_chinese_chars=A_				,				strip_accents=A_				,				**A_				,				)

									lowerCAmelCase_			:     Optional[int]									= json.loads(self.backend_tokenizer.normalizer.__getstate__())
									if (
									    normalizer_state.get('''lowercase'''				,				A_) != do_lower_case
									    or normalizer_state.get('''strip_accents'''				,				A_) != strip_accents
									    or normalizer_state.get('''handle_chinese_chars'''				,				A_) != tokenize_chinese_chars
									):
											lowerCAmelCase_			:     int									= getattr(A_				,				normalizer_state.pop('''type'''))
											lowerCAmelCase_			:     str									= do_lower_case
											lowerCAmelCase_			:     Dict									= strip_accents
											lowerCAmelCase_			:     Optional[Any]									= tokenize_chinese_chars
											lowerCAmelCase_			:     Union[str, Any]									= normalizer_class(**A_)

									lowerCAmelCase_			:     Any									= do_lower_case




							def    UpperCAmelCase__			(       self     :       Optional[Any]				,				A_     :       Optional[Any]				,				**A_     :       Tuple):
									lowerCAmelCase_			:     List[str]									= PaddingStrategy.MAX_LENGTH

									lowerCAmelCase_			:     str									= text
									lowerCAmelCase_			:     int									= kwargs.pop('''text_pair'''				,				A_)
									lowerCAmelCase_			:     str									= kwargs.pop('''return_tensors'''				,				A_)

									lowerCAmelCase_			:     int									= {
									    '''input_ids''': [],
									    '''attention_mask''': [],
									    '''token_type_ids''': [],
									}

									for idx, candidate_text in enumerate(A_):
											if batch_text_pair is not None:
													lowerCAmelCase_			:     List[Any]									= batch_text_pair[idx]
											else:
													lowerCAmelCase_			:     List[Any]									= None

											lowerCAmelCase_			:     int									= super().__call__(A_				,				A_				,				return_tensors=A_				,				**A_)

											lowerCAmelCase_			:     Optional[Any]									= encoded_candidates.get('''input_ids''')
											lowerCAmelCase_			:     List[str]									= encoded_candidates.get('''attention_mask''')
											lowerCAmelCase_			:     Optional[Any]									= encoded_candidates.get('''token_type_ids''')

											if encoded_input_ids is not None:
													output_data["input_ids"].append(A_)
											if encoded_attention_mask is not None:
													output_data["attention_mask"].append(A_)
											if encoded_token_type_ids is not None:
													output_data["token_type_ids"].append(A_)

									lowerCAmelCase_			:     List[str]									= {key: item for key, item in output_data.items() if len(A_) != 0}

									return BatchEncoding(A_				,				tensor_type=A_)




							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       Tuple				,				A_     :       List[Any]=None):
									lowerCAmelCase_			:     Optional[Any]									= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

									return output




							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       List[int]				,				A_     :       Optional[List[int]] = None):
									lowerCAmelCase_			:     Tuple									= [self.sep_token_id]
									lowerCAmelCase_			:     Union[str, Any]									= [self.cls_token_id]
									if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]
									return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       str				,				A_     :       Optional[str] = None):
									lowerCAmelCase_			:     List[str]									= self._tokenizer.model.save(A_				,				name=A_)
									return tuple(A_)





