


"""simple docstring"""




from manim import *
class    lowerCAmelCase      (   __a       ):




		'''simple docstring'''







		def 				__A		(      self							)	->				Union[str, Any]:
								SCREAMING_SNAKE_CASE						=   Rectangle(height=0.5		,			width=0.5							)
								SCREAMING_SNAKE_CASE						=   Rectangle(height=0.46		,			width=0.46							).set_stroke(width=0							)

								SCREAMING_SNAKE_CASE						=   [mem.copy() for i in range(6							)]
								SCREAMING_SNAKE_CASE						=   [mem.copy() for i in range(6							)]
								SCREAMING_SNAKE_CASE						=   VGroup(*__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0							)
								SCREAMING_SNAKE_CASE						=   VGroup(*__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0							)
								SCREAMING_SNAKE_CASE						=   VGroup(__UpperCamelCase		,			__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0							)
								SCREAMING_SNAKE_CASE						=   Text('CPU'		,			font_size=24							)
								SCREAMING_SNAKE_CASE						=   Group(__UpperCamelCase		,			__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0.5		,			aligned_edge=__UpperCamelCase							)
								cpu.move_to([-2.5, -0.5, 0]							)
								self.add(__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   [mem.copy() for i in range(4							)]
								SCREAMING_SNAKE_CASE						=   VGroup(*__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0							)
								SCREAMING_SNAKE_CASE						=   Text('GPU'		,			font_size=24							)
								SCREAMING_SNAKE_CASE						=   Group(__UpperCamelCase		,			__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0.5		,			aligned_edge=__UpperCamelCase							)
								gpu.move_to([-1, -1, 0]							)
								self.add(__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   [mem.copy() for i in range(6							)]
								SCREAMING_SNAKE_CASE						=   VGroup(*__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0							)

								SCREAMING_SNAKE_CASE						=   Text('Model'		,			font_size=24							)
								SCREAMING_SNAKE_CASE						=   Group(__UpperCamelCase		,			__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0.5		,			aligned_edge=__UpperCamelCase							)
								model.move_to([3, -1.0, 0]							)
								self.add(__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   []
								for i, rect in enumerate(__UpperCamelCase							):
														rect.set_stroke(__UpperCamelCase							)
														# target = fill.copy().set_fill(YELLOW, opacity=0.7)
														# target.move_to(rect)
														# self.add(target)

														SCREAMING_SNAKE_CASE						=   Rectangle(height=0.46 / 4		,			width=0.46 / 3							).set_stroke(width=0.0							).set_fill(__UpperCamelCase		,			opacity=0.7							)

														if i == 0:
																				cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT							)		,			buff=0.02		,			direction=__UpperCamelCase							)
																				cpu_target.set_x(cpu_target.get_x() + 0.1							)
														elif i == 3:
																				cpu_target.next_to(cpu_targs[0]		,			direction=__UpperCamelCase		,			buff=0.0							)
														else:
																				cpu_target.next_to(cpu_targs[i - 1]		,			direction=__UpperCamelCase		,			buff=0.0							)
														self.add(__UpperCamelCase							)
														cpu_targs.append(__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   [mem.copy() for i in range(6							)]
								SCREAMING_SNAKE_CASE						=   VGroup(*__UpperCamelCase							).arrange(__UpperCamelCase		,			buff=0							)

								SCREAMING_SNAKE_CASE						=   Text('Loaded Checkpoint'		,			font_size=24							)
								SCREAMING_SNAKE_CASE						=   Group(__UpperCamelCase		,			__UpperCamelCase							).arrange(__UpperCamelCase		,			aligned_edge=__UpperCamelCase		,			buff=0.4							)
								checkpoint.move_to([3, 0.5, 0]							)

								SCREAMING_SNAKE_CASE						=   Square(side_length=2.2							)
								key.move_to([-5, 2, 0]							)

								SCREAMING_SNAKE_CASE						=   MarkupText(
								    F'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'		,			font_size=18		,			)

								key_text.move_to([-5, 2.4, 0]							)

								self.add(__UpperCamelCase		,			__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   MarkupText(
								    F'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'		,			font_size=18		,			)

								blue_text.next_to(__UpperCamelCase		,			DOWN * 2.4		,			aligned_edge=key_text.get_left()							)

								SCREAMING_SNAKE_CASE						=   MarkupText(
								    F'Next, a <i><span fgcolor="{BLUE}">second</span></i> model is loaded into memory,\nwith the weights of a <span fgcolor="{BLUE}">single shard</span>.'		,			font_size=24		,			)
								step_a.move_to([2, 2, 0]							)
								self.play(Write(__UpperCamelCase							)		,			Write(__UpperCamelCase							)							)

								self.play(Write(__UpperCamelCase		,			run_time=1							)		,			Create(__UpperCamelCase		,			run_time=1							)							)

								SCREAMING_SNAKE_CASE						=   []
								SCREAMING_SNAKE_CASE						=   []
								for i, rect in enumerate(__UpperCamelCase							):
														SCREAMING_SNAKE_CASE						=   fill.copy().set_fill(__UpperCamelCase		,			opacity=0.7							)
														target.move_to(__UpperCamelCase							)
														first_animations.append(GrowFromCenter(__UpperCamelCase		,			run_time=1							)							)

														SCREAMING_SNAKE_CASE						=   target.copy()
														cpu_target.generate_target()
														if i < 5:
																				cpu_target.target.move_to(cpu_left_col_base[i + 1]							)
														else:
																				cpu_target.target.move_to(cpu_right_col_base[i - 5]							)
														second_animations.append(MoveToTarget(__UpperCamelCase		,			run_time=1.5							)							)

								self.play(*__UpperCamelCase							)
								self.play(*__UpperCamelCase							)
								self.wait()



from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE		:     int         =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	{
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}

class        UpperCamelCase						(				__a				):
						a__		:List[str]      =			'''megatron-bert'''





						def __init__(self			,     __UpperCamelCase=29_056			,     __UpperCamelCase=1_024			,     __UpperCamelCase=24			,     __UpperCamelCase=16			,     __UpperCamelCase=4_096			,     __UpperCamelCase="gelu"			,     __UpperCamelCase=0.1			,     __UpperCamelCase=0.1			,     __UpperCamelCase=512			,     __UpperCamelCase=2			,     __UpperCamelCase=0.02			,     __UpperCamelCase=1E-1_2			,     __UpperCamelCase=0			,     __UpperCamelCase="absolute"			,     __UpperCamelCase=True			,     **__UpperCamelCase			,     )				->				List[Any]:
												super().__init__(pad_token_id=__UpperCamelCase			,     **__UpperCamelCase						)

												UpperCamelCase_		:							Union[str, Any]  =		vocab_size
												UpperCamelCase_		:							Union[str, Any]  =		hidden_size
												UpperCamelCase_		:							Any  =		num_hidden_layers
												UpperCamelCase_		:							str  =		num_attention_heads
												UpperCamelCase_		:							List[Any]  =		hidden_act
												UpperCamelCase_		:							Dict  =		intermediate_size
												UpperCamelCase_		:							Optional[int]  =		hidden_dropout_prob
												UpperCamelCase_		:							Dict  =		attention_probs_dropout_prob
												UpperCamelCase_		:							Any  =		max_position_embeddings
												UpperCamelCase_		:							str  =		type_vocab_size
												UpperCamelCase_		:							Tuple  =		initializer_range
												UpperCamelCase_		:							List[str]  =		layer_norm_eps
												UpperCamelCase_		:							Tuple  =		position_embedding_type
												UpperCamelCase_		:							Tuple  =		use_cache






