



from manim import *




class 					snake_case__						(       lowerCAmelCase_     ):






     """simple docstring"""


     def    lowercase_   (       self  :  int					)   ->Any:
            snake_case__							:	Union[str, Any]   	=	Rectangle(height=0.5, width=0.5					)
            snake_case__							:	Tuple   	=	Rectangle(height=0.4_6, width=0.4_6					).set_stroke(width=0					)

            snake_case__							:	Tuple   	=	[mem.copy() for i in range(6					)]
            snake_case__							:	List[str]   	=	[mem.copy() for i in range(6					)]
            snake_case__							:	Any   	=	VGroup(*_snake_case					).arrange(_snake_case, buff=0					)
            snake_case__							:	Dict   	=	VGroup(*_snake_case					).arrange(_snake_case, buff=0					)
            snake_case__							:	int   	=	VGroup(_snake_case, _snake_case					).arrange(_snake_case, buff=0					)
            snake_case__							:	Dict   	=	Text('CPU', font_size=2_4					)
            snake_case__							:	Union[str, Any]   	=	Group(_snake_case, _snake_case					).arrange(_snake_case, buff=0.5, aligned_edge=_snake_case					)
            cpu.move_to([-2.5, -0.5, 0]					)
            self.add(_snake_case					)

            snake_case__							:	int   	=	[mem.copy() for i in range(1					)]
            snake_case__							:	int   	=	VGroup(*_snake_case					).arrange(_snake_case, buff=0					)
            snake_case__							:	Tuple   	=	Text('GPU', font_size=2_4					)
            snake_case__							:	int   	=	Group(_snake_case, _snake_case					).arrange(_snake_case, buff=0.5, aligned_edge=_snake_case					)
            gpu.align_to(_snake_case, _snake_case					)
            gpu.set_x(gpu.get_x() - 1					)

            self.add(_snake_case					)

            snake_case__							:	Tuple   	=	[mem.copy() for i in range(6					)]
            snake_case__							:	List[str]   	=	VGroup(*_snake_case					).arrange(_snake_case, buff=0					)

            snake_case__							:	Union[str, Any]   	=	Text('Model', font_size=2_4					)
            snake_case__							:	Optional[Any]   	=	Group(_snake_case, _snake_case					).arrange(_snake_case, buff=0.5, aligned_edge=_snake_case					)
            model.move_to([3, -1.0, 0]					)

            self.play(
                Create(_snake_case, run_time=1					), Create(_snake_case, run_time=1					), Create(_snake_case, run_time=1					), )

            snake_case__							:	Union[str, Any]   	=	MarkupText(
                F'''First, an empty model skeleton is loaded\ninto <span fgcolor=\'{YELLOW}\'>memory</span> without using much RAM.''', font_size=2_4, )

            snake_case__							:	int   	=	Square(side_length=2.2					)
            key.move_to([-5, 2, 0]					)

            snake_case__							:	str   	=	MarkupText(
                F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model''', font_size=1_8, )

            key_text.move_to([-5, 2.4, 0]					)

            step_a.move_to([2, 2, 0]					)
            self.play(Write(_snake_case, run_time=2.5					), Write(_snake_case					), Write(_snake_case					)					)

            self.add(_snake_case					)

            snake_case__							:	List[Any]   	=	[]
            snake_case__							:	str   	=	[]
            snake_case__							:	Optional[int]   	=	[]
            for i, rect in enumerate(_snake_case					):
                   snake_case__							:	str   	=	Rectangle(height=0.4_6, width=0.4_6					).set_stroke(width=0.0					).set_fill(_snake_case, opacity=0.7					)
                   cpu_target.move_to(_snake_case					)
                   cpu_target.generate_target()
                   snake_case__							:	str   	=	0.4_6 / 4
                   snake_case__							:	Optional[int]   	=	0.4_6 / 3

                   if i == 0:
                          cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT					), buff=0.0_2, direction=_snake_case					)
                          cpu_target.target.set_x(cpu_target.target.get_x() + 0.1					)
                   elif i == 3:
                          cpu_target.target.next_to(cpu_targs[0].target, direction=_snake_case, buff=0.0					)
                   else:
                          cpu_target.target.next_to(cpu_targs[i - 1].target, direction=_snake_case, buff=0.0					)
                   cpu_targs.append(_snake_case					)

                   first_animations.append(rect.animate(run_time=0.5					).set_stroke(_snake_case					)					)
                   second_animations.append(MoveToTarget(_snake_case, run_time=1.5					)					)

            self.play(*_snake_case					)
            self.play(*_snake_case					)

            self.wait()











import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin




class 					snake_case__						(       lowerCAmelCase_				,     unittest.TestCase     ):






     """simple docstring"""


     _SCREAMING_SNAKE_CASE						=	TransfoXLTokenizer
     _SCREAMING_SNAKE_CASE						=	False
     _SCREAMING_SNAKE_CASE						=	False






     def    lowercase_   (       self  :  Optional[int]					)   ->Any:
            super().setUp()

            snake_case__							:	Tuple   	=	[
                '<unk>',
                '[CLS]',
                '[SEP]',
                'want',
                'unwanted',
                'wa',
                'un',
                'running',
                ',',
                'low',
                'l',
            ]
            snake_case__							:	Any   	=	os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file']					)
            with open(self.vocab_file, 'w', encoding='utf-8'					) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)






     def    lowercase_   (       self  :  Union[str, Any], **_snake_case  :  List[Any]					)   ->Dict:
            snake_case__							:	str   	=	True
            return TransfoXLTokenizer.from_pretrained(self.tmpdirname, **_snake_case					)






     def    lowercase_   (       self  :  Optional[Any], _snake_case  :  str					)   ->Dict:
            snake_case__							:	List[Any]   	=	'<unk> UNwanted , running'
            snake_case__							:	List[Any]   	=	'<unk> unwanted, running'
            return input_text, output_text






     def    lowercase_   (       self  :  List[Any]					)   ->Tuple:
            snake_case__							:	Dict   	=	TransfoXLTokenizer(vocab_file=self.vocab_file, lower_case=_snake_case					)

            snake_case__							:	str   	=	tokenizer.tokenize('<unk> UNwanted , running'					)
            self.assertListEqual(_snake_case, ['<unk>', 'unwanted', ',', 'running']					)

            self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case					), [0, 4, 8, 7]					)






     def    lowercase_   (       self  :  List[str]					)   ->List[Any]:
            snake_case__							:	str   	=	TransfoXLTokenizer(lower_case=_snake_case					)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					), ['hello', '!', 'how', 'are', 'you', '?']					)






     def    lowercase_   (       self  :  Optional[int]					)   ->Optional[Any]:
            snake_case__							:	Optional[int]   	=	TransfoXLTokenizer(lower_case=_snake_case					)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					), ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






     def    lowercase_   (       self  :  Optional[int]					)   ->Union[str, Any]:
            snake_case__							:	List[Any]   	=	TransfoXLTokenizer(lower_case=_snake_case					)
            snake_case__							:	Dict   	=	'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
            snake_case__							:	List[Any]   	=	[
                'Hello',
                '(',
                'bracket',
                ')',
                'and',
                'side',
                '@-@',
                'scrolled',
                '[',
                'and',
                ']',
                'Henry',
                '\'s',
                '$',
                '5',
                '@,@',
                '000',
                'with',
                '3',
                '@.@',
                '34',
                'm',
                '.',
                'What',
                '\'s',
                'up',
                '!',
                '?',
            ]

            self.assertListEqual(tokenizer.tokenize(_snake_case					), _snake_case					)

            self.assertEqual(tokenizer.convert_tokens_to_string(_snake_case					), _snake_case					)






     def    lowercase_   (       self  :  Dict					)   ->Any:
            snake_case__							:	Dict   	=	self.get_tokenizer()
            snake_case__							:	Optional[Any]   	=	len(_snake_case					)

            tokenizer.add_tokens(['new1', 'new2']					)
            tokenizer.move_added_token('new1', 1					)

            # Check that moved token is not copied (duplicate)
            self.assertEqual(len(_snake_case					), original_len + 2					)
            # Check that token is moved to specified id
            self.assertEqual(tokenizer.encode('new1'					), [1]					)
            self.assertEqual(tokenizer.decode([1]					), 'new1'					)






