






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam

class 		_snake_case	(	datasets.BeamBasedBuilder       ):






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							int:
						return datasets.DatasetInfo(
						    features=datasets.Features({'content': datasets.Value('string')})		,       supervised_keys=a		,       )






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a)   ->							Any:
						return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,       gen_kwargs={'examples': get_test_dummy_examples()})]







			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a)   ->							Union[str, Any]:
						import apache_beam as beam

						return pipeline | "Load Examples" >> beam.Create(a)



class 		_snake_case	(	datasets.BeamBasedBuilder       ):






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							str:
						return datasets.DatasetInfo(
						    features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string')})})		,       supervised_keys=a		,       )






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a)   ->							Union[str, Any]:
						return [
						    datasets.SplitGenerator(name=datasets.Split.TRAIN		,       gen_kwargs={'examples': get_test_nested_examples()})
						]







			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a)   ->							int:
						import apache_beam as beam

						return pipeline | "Load Examples" >> beam.Create(a)






def 					lowerCamelCase__					():
			return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar'])]






def 					lowerCamelCase__					():
			return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar'])]







class 		_snake_case	(	A__       ):






			@require_beam
			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Dict:
						SCREAMING_SNAKE_CASE							=			len(get_test_dummy_examples())
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
									SCREAMING_SNAKE_CASE							=			DummyBeamDataset(cache_dir=a		,       beam_runner='DirectRunner')
									builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(a		,       builder.name		,       'default'		,       '0.0.0'		,       f'''{builder.name}-train.arrow''')))
									self.assertDictEqual(builder.info.features		,       datasets.Features({'content': datasets.Value('string')}))
									SCREAMING_SNAKE_CASE							=			builder.as_dataset()
									self.assertEqual(dset['train'].num_rows		,       a)
									self.assertEqual(dset['train'].info.splits['train'].num_examples		,       a)
									self.assertDictEqual(dset['train'][0]		,       get_test_dummy_examples()[0][1])
									self.assertDictEqual(
									    dset['train'][expected_num_examples - 1]		,       get_test_dummy_examples()[expected_num_examples - 1][1])
									self.assertTrue(
									    os.path.exists(os.path.join(a		,       builder.name		,       'default'		,       '0.0.0'		,       'dataset_info.json')))
									del dset






			@require_beam
			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Union[str, Any]:
						import apache_beam as beam

						SCREAMING_SNAKE_CASE							=			beam.io.parquetio.WriteToParquet

						SCREAMING_SNAKE_CASE							=			len(get_test_dummy_examples())
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
									SCREAMING_SNAKE_CASE							=			DummyBeamDataset(cache_dir=a		,       beam_runner='DirectRunner')
									with patch('apache_beam.io.parquetio.WriteToParquet') as write_parquet_mock:
												SCREAMING_SNAKE_CASE							=			partial(a		,       num_shards=2)
												builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(
									            a		,       builder.name		,       'default'		,       '0.0.0'		,       f'''{builder.name}-train-00000-of-00002.arrow''')))
									self.assertTrue(
									    os.path.exists(
									        os.path.join(
									            a		,       builder.name		,       'default'		,       '0.0.0'		,       f'''{builder.name}-train-00000-of-00002.arrow''')))
									self.assertDictEqual(builder.info.features		,       datasets.Features({'content': datasets.Value('string')}))
									SCREAMING_SNAKE_CASE							=			builder.as_dataset()
									self.assertEqual(dset['train'].num_rows		,       a)
									self.assertEqual(dset['train'].info.splits['train'].num_examples		,       a)
									# Order is not preserved when sharding, so we just check that all the elements are there
									self.assertListEqual(sorted(dset['train']['content'])		,       sorted(['foo', 'bar', 'foobar']))
									self.assertTrue(
									    os.path.exists(os.path.join(a		,       builder.name		,       'default'		,       '0.0.0'		,       'dataset_info.json')))
									del dset






			@require_beam
			def  SCREAMING_SNAKE_CASE__     (							self)   ->							List[str]:
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
									SCREAMING_SNAKE_CASE							=			DummyBeamDataset(cache_dir=a)
									self.assertRaises(datasets.builder.MissingBeamOptions		,       builder.download_and_prepare)







			@require_beam
			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Dict:
						SCREAMING_SNAKE_CASE							=			len(get_test_nested_examples())
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
									SCREAMING_SNAKE_CASE							=			NestedBeamDataset(cache_dir=a		,       beam_runner='DirectRunner')
									builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(a		,       builder.name		,       'default'		,       '0.0.0'		,       f'''{builder.name}-train.arrow''')))
									self.assertDictEqual(
									    builder.info.features		,       datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string')})}))
									SCREAMING_SNAKE_CASE							=			builder.as_dataset()
									self.assertEqual(dset['train'].num_rows		,       a)
									self.assertEqual(dset['train'].info.splits['train'].num_examples		,       a)
									self.assertDictEqual(dset['train'][0]		,       get_test_nested_examples()[0][1])
									self.assertDictEqual(
									    dset['train'][expected_num_examples - 1]		,       get_test_nested_examples()[expected_num_examples - 1][1])
									self.assertTrue(
									    os.path.exists(os.path.join(a		,       builder.name		,       'default'		,       '0.0.0'		,       'dataset_info.json')))
									del dset











from manim import *

class 							__SCREAMING_SNAKE_CASE       (  _a							):






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      Rectangle(height=0.5	,			width=0.5	)
														UpperCamelCase__					  =      Rectangle(height=0.25	,			width=0.25	)
														UpperCamelCase__					  =      Rectangle(height=0.46	,			width=0.46	).set_stroke(width=0	)

														UpperCamelCase__					  =      [mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      [mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      VGroup(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      Text("""CPU"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														cpu.move_to([-2.5, -0.5, 0]	)
														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      [mem.copy() for i in range(4	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      Text("""GPU"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														gpu.move_to([-1, -1, 0]	)
														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      [mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)

														UpperCamelCase__					  =      Text("""Model"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														model.move_to([3, -1.0, 0]	)
														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      []
														UpperCamelCase__					  =      []
														UpperCamelCase__					  =      []

														for i, rect in enumerate(__lowerCAmelCase	):
																					rect.set_stroke(__lowerCAmelCase	)

																					UpperCamelCase__					  =      Rectangle(height=0.46 / 4	,			width=0.46 / 3	).set_stroke(width=0.0	).set_fill(__lowerCAmelCase	,			opacity=0.7	)

																					if i == 0:
																												cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT	)	,			buff=0.02	,			direction=__lowerCAmelCase	)
																												cpu_target.set_x(cpu_target.get_x() + 0.1	)
																					elif i == 3:
																												cpu_target.next_to(model_cpu_arr[0]	,			direction=__lowerCAmelCase	,			buff=0.0	)
																					else:
																												cpu_target.next_to(model_cpu_arr[i - 1]	,			direction=__lowerCAmelCase	,			buff=0.0	)
																					self.add(__lowerCAmelCase	)
																					model_cpu_arr.append(__lowerCAmelCase	)

														self.add(*__lowerCAmelCase	,			*__lowerCAmelCase	,			*__lowerCAmelCase	)

														UpperCamelCase__					  =      [mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)

														UpperCamelCase__					  =      Text("""Loaded Checkpoint"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														checkpoint.move_to([3, 0.5, 0]	)

														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      []
														UpperCamelCase__					  =      []

														for i, rect in enumerate(__lowerCAmelCase	):
																					UpperCamelCase__					  =      fill.copy().set_fill(__lowerCAmelCase	,			opacity=0.7	)
																					target.move_to(__lowerCAmelCase	)
																					ckpt_arr.append(__lowerCAmelCase	)

																					UpperCamelCase__					  =      target.copy()
																					if i < 5:
																												cpu_target.move_to(cpu_left_col_base[i + 1]	)
																					else:
																												cpu_target.move_to(cpu_right_col_base[i - 5]	)
																					ckpt_cpu_arr.append(__lowerCAmelCase	)
														self.add(*__lowerCAmelCase	,			*__lowerCAmelCase	)

														UpperCamelCase__					  =      Square(side_length=2.2	)
														key.move_to([-5, 2, 0]	)

														UpperCamelCase__					  =      MarkupText(
														    f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""	,			font_size=18	,			)

														key_text.move_to([-5, 2.4, 0]	)

														self.add(__lowerCAmelCase	,			__lowerCAmelCase	)

														UpperCamelCase__					  =      MarkupText(
														    f"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""	,			font_size=18	,			)

														blue_text.next_to(__lowerCAmelCase	,			DOWN * 2.4	,			aligned_edge=key_text.get_left()	)
														self.add(__lowerCAmelCase	)

														UpperCamelCase__					  =      MarkupText(
														    f"""Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device."""	,			font_size=24	,			)
														step_a.move_to([2, 2, 0]	)

														UpperCamelCase__					  =      [meta_mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      [meta_mem.copy() for i in range(6	)]
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      VGroup(*__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      VGroup(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0	)
														UpperCamelCase__					  =      Text("""Disk"""	,			font_size=24	)
														UpperCamelCase__					  =      Group(__lowerCAmelCase	,			__lowerCAmelCase	).arrange(__lowerCAmelCase	,			buff=0.5	,			aligned_edge=__lowerCAmelCase	)
														disk.move_to([-4.0, -1.25, 0]	)
														self.play(Write(__lowerCAmelCase	,			run_time=3	)	,			Write(__lowerCAmelCase	,			run_time=1	)	,			Create(__lowerCAmelCase	,			run_time=1	)	)

														UpperCamelCase__					  =      []
														for i, rect in enumerate(__lowerCAmelCase	):
																					UpperCamelCase__					  =      rect.copy()
																					target.generate_target()
																					target.target.move_to(disk_left_col_base[i]	).scale(0.5	)
																					animations.append(MoveToTarget(__lowerCAmelCase	,			run_time=1.5	)	)
														self.play(*__lowerCAmelCase	)

														self.play(FadeOut(__lowerCAmelCase	)	)

														UpperCamelCase__					  =      MarkupText(f"""Then, the checkpoint is removed from memory\nthrough garbage collection."""	,			font_size=24	)
														step_a.move_to([2, 2, 0]	)

														self.play(Write(__lowerCAmelCase	,			run_time=3	)	)

														self.play(
														    FadeOut(__lowerCAmelCase	,			__lowerCAmelCase	,			*__lowerCAmelCase	,			*__lowerCAmelCase	)	,			)

														self.wait()




