






from manim import *





class     SCREAMING_SNAKE_CASE					(			UpperCAmelCase__       ):





							"""simple docstring"""

							def    __A			(	self: List[str] )     ->					Optional[int]:
												_A										=			Rectangle(height=0.5		,  width=0.5 )
												_A										=			Rectangle(height=0.46		,  width=0.46 ).set_stroke(width=0 )

												_A										=			Rectangle(height=0.25		,  width=0.25 )

												_A										=			[mem.copy() for i in range(6 )]
												_A										=			[mem.copy() for i in range(6 )]
												_A										=			VGroup(*lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0 )
												_A										=			VGroup(*lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0 )
												_A										=			VGroup(lowerCamelCase__		,  lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0 )
												_A										=			Text('''CPU'''		,  font_size=24 )
												_A										=			Group(lowerCamelCase__		,  lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0.5		,  aligned_edge=lowerCamelCase__ )
												cpu.move_to([-2.5, -0.5, 0] )
												self.add(lowerCamelCase__ )

												_A										=			[mem.copy() for i in range(4 )]
												_A										=			VGroup(*lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0 )
												_A										=			Text('''GPU'''		,  font_size=24 )
												_A										=			Group(lowerCamelCase__		,  lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0.5		,  aligned_edge=lowerCamelCase__ )
												gpu.move_to([-1, -1, 0] )
												self.add(lowerCamelCase__ )

												_A										=			[mem.copy() for i in range(6 )]
												_A										=			VGroup(*lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0 )

												_A										=			Text('''Model'''		,  font_size=24 )
												_A										=			Group(lowerCamelCase__		,  lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0.5		,  aligned_edge=lowerCamelCase__ )
												model.move_to([3, -1.0, 0] )
												self.add(lowerCamelCase__ )

												_A										=			[]
												_A										=			[]

												for i, rect in enumerate(lowerCamelCase__ ):
																	_A										=			fill.copy().set_fill(lowerCamelCase__		,  opacity=0.8 )
																	target.move_to(lowerCamelCase__ )
																	model_arr.append(lowerCamelCase__ )

																	_A										=			Rectangle(height=0.46		,  width=0.46 ).set_stroke(width=0.0 ).set_fill(lowerCamelCase__		,  opacity=0.8 )
																	cpu_target.move_to(cpu_left_col_base[i] )
																	model_cpu_arr.append(lowerCamelCase__ )

												self.add(*lowerCamelCase__		,  *lowerCamelCase__ )

												_A										=			[meta_mem.copy() for i in range(6 )]
												_A										=			[meta_mem.copy() for i in range(6 )]
												_A										=			VGroup(*lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0 )
												_A										=			VGroup(*lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0 )
												_A										=			VGroup(lowerCamelCase__		,  lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0 )
												_A										=			Text('''Disk'''		,  font_size=24 )
												_A										=			Group(lowerCamelCase__		,  lowerCamelCase__ ).arrange(lowerCamelCase__		,  buff=0.5		,  aligned_edge=lowerCamelCase__ )
												disk.move_to([-4, -1.25, 0] )
												self.add(lowerCamelCase__		,  lowerCamelCase__ )

												_A										=			Square(side_length=2.2 )
												key.move_to([-5, 2, 0] )

												_A										=			MarkupText(
												    f"""<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model"""		,  font_size=18		,  )

												key_text.move_to([-5, 2.4, 0] )

												self.add(lowerCamelCase__		,  lowerCamelCase__ )

												_A										=			MarkupText(
												    f"""<span fgcolor=\'{BLUE}\'>●</span> Checkpoint"""		,  font_size=18		,  )

												blue_text.next_to(lowerCamelCase__		,  DOWN * 2.4		,  aligned_edge=key_text.get_left() )
												self.add(lowerCamelCase__ )

												_A										=			MarkupText(
												    f"""Now watch as an input is passed through the model\nand how the memory is utilized and handled."""		,  font_size=24		,  )
												step_a.move_to([2, 2, 0] )

												self.play(Write(lowerCamelCase__ ) )

												_A										=			Square(0.3 )
												input.set_fill(lowerCamelCase__		,  opacity=1.0 )
												input.set_stroke(width=0.0 )
												input.next_to(model_base[0]		,  lowerCamelCase__		,  buff=0.5 )

												self.play(Write(lowerCamelCase__ ) )

												input.generate_target()
												input.target.next_to(model_arr[0]		,  direction=lowerCamelCase__		,  buff=0.02 )
												self.play(MoveToTarget(lowerCamelCase__ ) )

												self.play(FadeOut(lowerCamelCase__ ) )

												_A										=			Arrow(start=lowerCamelCase__		,  end=lowerCamelCase__		,  color=lowerCamelCase__		,  buff=0.5 )
												a.next_to(model_arr[0].get_left()		,  lowerCamelCase__		,  buff=0.2 )

												model_cpu_arr[0].generate_target()
												model_cpu_arr[0].target.move_to(gpu_rect[0] )

												_A										=			MarkupText(
												    f"""As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back."""		,  font_size=24		,  )
												step_a.move_to([2, 2, 0] )

												self.play(Write(lowerCamelCase__		,  run_time=3 ) )

												_A										=			{"run_time": 1, "fade_in": True, "fade_out": True, "buff": 0.02}

												self.play(
												    Write(lowerCamelCase__ )		,  Circumscribe(model_arr[0]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  Circumscribe(model_cpu_arr[0]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  Circumscribe(gpu_rect[0]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  )
												self.play(MoveToTarget(model_cpu_arr[0] ) )

												_A										=			a.copy()
												for i in range(6 ):
																	a_c.next_to(model_arr[i].get_right() + 0.02		,  lowerCamelCase__		,  buff=0.2 )

																	input.generate_target()
																	input.target.move_to(model_arr[i].get_right() + 0.02 )

																	_A										=			AnimationGroup(
																	    FadeOut(lowerCamelCase__		,  run_time=0.5 )		,  MoveToTarget(lowerCamelCase__		,  run_time=0.5 )		,  FadeIn(lowerCamelCase__		,  run_time=0.5 )		,  lag_ratio=0.2 )

																	self.play(lowerCamelCase__ )

																	model_cpu_arr[i].generate_target()
																	model_cpu_arr[i].target.move_to(cpu_left_col_base[i] )

																	if i < 5:
																						model_cpu_arr[i + 1].generate_target()
																						model_cpu_arr[i + 1].target.move_to(gpu_rect[0] )
																						if i >= 1:
																											_A										=			0.7

																						self.play(
																						    Circumscribe(model_arr[i]		,  **lowerCamelCase__ )		,  Circumscribe(cpu_left_col_base[i]		,  **lowerCamelCase__ )		,  Circumscribe(cpu_left_col_base[i + 1]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  Circumscribe(gpu_rect[0]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  Circumscribe(model_arr[i + 1]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  )
																						if i < 1:
																											self.play(
																											    MoveToTarget(model_cpu_arr[i] )		,  MoveToTarget(model_cpu_arr[i + 1] )		,  )
																						else:
																											self.play(
																											    MoveToTarget(model_cpu_arr[i]		,  run_time=0.7 )		,  MoveToTarget(model_cpu_arr[i + 1]		,  run_time=0.7 )		,  )
																	else:
																						model_cpu_arr[i].generate_target()
																						model_cpu_arr[i].target.move_to(cpu_left_col_base[-1] )
																						input.generate_target()
																						input.target.next_to(model_arr[-1].get_right()		,  RIGHT + 0.02		,  buff=0.2 )

																						self.play(
																						    Circumscribe(model_arr[-1]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  Circumscribe(cpu_left_col_base[-1]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  Circumscribe(gpu_rect[0]		,  color=lowerCamelCase__		,  **lowerCamelCase__ )		,  )

																						self.play(MoveToTarget(model_cpu_arr[i] ) )

																	_A										=			a_c
																	_A										=			a_c.copy()

												input.generate_target()
												input.target.next_to(model_base[-1]		,  RIGHT + 0.02		,  buff=0.5 )
												self.play(
												    FadeOut(lowerCamelCase__ )		,  FadeOut(lowerCamelCase__		,  run_time=0.5 )		,  )

												_A										=			MarkupText(f"""Inference on a model too large for GPU memory\nis successfully completed."""		,  font_size=24 )
												step_a.move_to([2, 2, 0] )

												self.play(Write(lowerCamelCase__		,  run_time=3 )		,  MoveToTarget(lowerCamelCase__ ) )

												self.wait()




from ...configuration_utils import PretrainedConfig
from ...utils import logging


a  :      List[Any]            =    logging.get_logger(__name__)

a  :      Any            =    {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/config.json"""}

class 					_lowercase    (     UpperCAmelCase__			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				int          =    'openai-gpt'
    SCREAMING_SNAKE_CASE:				List[str]          =    {
        'max_position_embeddings': 'n_positions',
        'hidden_size': 'n_embd',
        'num_attention_heads': 'n_head',
        'num_hidden_layers': 'n_layer',
    }




    def __init__(    self       ,						lowerCamelCase__=40_478       ,						lowerCamelCase__=512       ,						lowerCamelCase__=768       ,						lowerCamelCase__=12       ,						lowerCamelCase__=12       ,						lowerCamelCase__="gelu"       ,						lowerCamelCase__=0.1       ,						lowerCamelCase__=0.1       ,						lowerCamelCase__=0.1       ,						lowerCamelCase__=1E-5       ,						lowerCamelCase__=0.0_2       ,						lowerCamelCase__="cls_index"       ,						lowerCamelCase__=True       ,						lowerCamelCase__=None       ,						lowerCamelCase__=True       ,						lowerCamelCase__=0.1       ,						**lowerCamelCase__       ,						):
       lowerCAmelCase_:							Union[str, Any]       =	vocab_size
       lowerCAmelCase_:							List[Any]       =	n_positions
       lowerCAmelCase_:							Tuple       =	n_embd
       lowerCAmelCase_:							Optional[int]       =	n_layer
       lowerCAmelCase_:							Optional[int]       =	n_head
       lowerCAmelCase_:							int       =	afn
       lowerCAmelCase_:							str       =	resid_pdrop
       lowerCAmelCase_:							Optional[int]       =	embd_pdrop
       lowerCAmelCase_:							Optional[int]       =	attn_pdrop
       lowerCAmelCase_:							Dict       =	layer_norm_epsilon
       lowerCAmelCase_:							List[Any]       =	initializer_range
       lowerCAmelCase_:							Union[str, Any]       =	summary_type
       lowerCAmelCase_:							Any       =	summary_use_proj
       lowerCAmelCase_:							Dict       =	summary_activation
       lowerCAmelCase_:							Dict       =	summary_first_dropout
       lowerCAmelCase_:							List[Any]       =	summary_proj_to_labels
       super().__init__(**lowerCamelCase__ )