
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_lowerCamelCase       :				int       =			logging.getLogger(__name__)






def        a_		(       ) ->  Union[str, Any]:
			_snake_case		=			argparse.ArgumentParser(
			    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.' )
			parser.add_argument(
			    '--dataset_name'       ,					type=__lowercase       ,					default='wikitext'       ,					help='Name of the training. Explore datasets at: hf.co/datasets.'       ,					)
			parser.add_argument(
			    '--dataset_config'       ,					type=__lowercase       ,					default='wikitext-103-raw-v1'       ,					help='Configuration name of the dataset.' )
			parser.add_argument(
			    '--tokenizer_name_or_path'       ,					type=__lowercase       ,					default='sayakpaul/unigram-tokenizer-wikitext'       ,					help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'       ,					)
			parser.add_argument(
			    '--shard_size'       ,					type=__lowercase       ,					default=1_000       ,					help='Number of entries to go in a single shard.'       ,					)
			parser.add_argument('--split'       ,					type=__lowercase       ,					default='train'       ,					choices=['train', 'test', 'validation'] )
			parser.add_argument(
			    '--limit'       ,					default=__lowercase       ,					type=__lowercase       ,					help='Limit the number of shards (used for debugging).'       ,					)
			parser.add_argument(
			    '--max_length'       ,					type=__lowercase       ,					default=512       ,					help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
			    ' sequence length that is a multiple of 8.'       ,					)
			parser.add_argument(
			    '--output_dir'       ,					default='tf-tpu'       ,					type=__lowercase       ,					help='Output directory where the TFRecord shards will be saved. If the'
			    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
			    ' shards will be directly saved to a Google Cloud Storage bucket.'       ,					)

			_snake_case		=			parser.parse_args()
			return args






def        a_		(       __lowercase   :	Any ) ->  str:
			def fn(__lowercase   :	Dict ):
						return tokenizer(examples['text'] )

			return fn






def        a_		(       __lowercase   :	Tuple ) ->  int:
			_snake_case		=			[]
			for i in range(len(tokenized_data['input_ids'] ) ):
						_snake_case		=			{
						    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i] ) ),
						    'attention_mask': tf.train.Feature(
						        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i] ) ),
						}
						_snake_case		=			tf.train.Features(feature=__lowercase )
						_snake_case		=			tf.train.Example(features=__lowercase )
						_snake_case		=			example.SerializeToString()
						records.append(__lowercase )
			return records






def        a_		(       __lowercase   :	Dict ) ->  Any:
			_snake_case		=			datasets.load_dataset(args.dataset_name       ,					args.dataset_config       ,					split=args.split )

			if args.limit is not None:
						_snake_case		=			min(len(__lowercase )       ,					args.limit )
						_snake_case		=			dataset.select(range(__lowercase ) )
						print(f'''Limiting the dataset to {args.limit} entries.''' )

			_snake_case		=			AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

			# Handle output directory creation.
			# For serializing into a Google Cloud Storage Bucket, one needs to first
			# create a bucket.
			if "gs" not in args.output_dir:
						if not os.path.exists(args.output_dir ):
									os.makedirs(args.output_dir )
						_snake_case		=			os.path.join(args.output_dir       ,					args.split )
						if not os.path.exists(__lowercase ):
									os.makedirs(__lowercase )
			else:
						_snake_case		=			os.path.join(args.output_dir       ,					args.split )

			# Tokenize the whole dataset at once.
			_snake_case		=			tokenize_function(__lowercase )
			_snake_case		=			dataset.map(__lowercase       ,					batched=__lowercase       ,					num_proc=4       ,					remove_columns=['text'] )

			# We need to concatenate all our texts together, and then split the result
			# into chunks of a fixed size, which we will call block_size. To do this, we
			# will use the map method again, with the option batched=True. When we use batched=True,
			# the function we pass to map() will be passed multiple inputs at once, allowing us
			# to group them into more or fewer examples than we had in the input.
			# This allows us to create our new fixed-length samples. The advantage of this
			# method is that we don't lose a whole lot of content from the dataset compared to the
			# case where we simply tokenize with a pre-defined max_length.

			def group_texts(__lowercase   :	Optional[int] ):
						# Concatenate all texts.
						_snake_case		=			{k: sum(examples[k]       ,					[] ) for k in examples.keys()}
						_snake_case		=			len(concatenated_examples[list(examples.keys() )[0]] )
						# We drop the small remainder, though you could add padding instead if the model supports it
						# In this, as in all things, we advise you to follow your heart 🫀
						_snake_case		=			(total_length // args.max_length) * args.max_length
						# Split by chunks of max_len.
						_snake_case		=			{
						    k: [t[i : i + args.max_length] for i in range(0       ,					__lowercase       ,					args.max_length )]
						    for k, t in concatenated_examples.items()
						}
						return result

			_snake_case		=			dataset_tokenized.map(__lowercase       ,					batched=__lowercase       ,					batch_size=1_000       ,					num_proc=4 )

			_snake_case		=			0
			_snake_case		=			0
			for shard in range(0       ,					len(__lowercase )       ,					args.shard_size ):
						_snake_case		=			grouped_dataset[shard : shard + args.shard_size]
						_snake_case		=			len(dataset_snapshot['input_ids'] )
						_snake_case		=			os.path.join(__lowercase       ,					f'''dataset-{shard_count}-{records_containing}.tfrecord''' )
						_snake_case		=			get_serialized_examples(__lowercase )

						with tf.io.TFRecordWriter(__lowercase ) as out_file:
									for i in range(len(__lowercase ) ):
												_snake_case		=			serialized_examples[i]
												out_file.write(__lowercase )
									print('Wrote file {} containing {} records'.format(__lowercase       ,					__lowercase ) )

						shard_count += 1
						total_records += records_containing

			with open(f'''split-{args.split}-records-count.txt'''       ,					'w' ) as f:
						print(f'''Total {args.split} records: {total_records}'''       ,					file=__lowercase )


if __name__ == "__main__":
			_lowerCamelCase       :				Union[str, Any]       =			parse_args()
			main(args)

import os
import tempfile
import unittest
from pathlib import Path

from transformers import AutoConfig, is_torch_available
from transformers.testing_utils import require_torch, torch_device


if is_torch_available():
			from transformers import PyTorchBenchmark, PyTorchBenchmarkArguments




@require_torch
class    SCREAMING_SNAKE_CASE__     (	unittest.TestCase					):





	'''simple docstring'''



	def  A							(  self	:    List[Any]			,  lowercase	:    Dict	):


				'''simple docstring'''


				for model_result in results.values():
							for batch_size, sequence_length in zip(model_result['bs']			,  model_result['ss']	):
										_snake_case		=			model_result['result'][batch_size][sequence_length]
										self.assertIsNotNone(lowercase	)



	def  A							(  self	:    str	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_inference_result	)
				self.check_results_dict_not_empty(results.memory_inference_result	)



	def  A							(  self	:    Any	):


				'''simple docstring'''


				_snake_case		=			'sgugger/tiny-distilbert-classification'
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  only_pretrain_model=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_inference_result	)
				self.check_results_dict_not_empty(results.memory_inference_result	)



	def  A							(  self	:    Optional[int]	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  torchscript=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_inference_result	)
				self.check_results_dict_not_empty(results.memory_inference_result	)



	@unittest.skipIf(torch_device == 'cpu'			,  'Cant do half precision'	)
	def  A							(  self	:    Optional[Any]	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  fpaa=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_inference_result	)
				self.check_results_dict_not_empty(results.memory_inference_result	)



	def  A							(  self	:    str	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				_snake_case		=			AutoConfig.from_pretrained(lowercase	)
				# set architectures equal to `None`
				_snake_case		=			None
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase			,  configs=[config]	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_inference_result	)
				self.check_results_dict_not_empty(results.memory_inference_result	)



	def  A							(  self	:    Optional[Any]	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_train_result	)
				self.check_results_dict_not_empty(results.memory_train_result	)



	@unittest.skipIf(torch_device == 'cpu'			,  'Can\'t do half precision'	)
	def  A							(  self	:    str	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  fpaa=lowercase			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_train_result	)
				self.check_results_dict_not_empty(results.memory_train_result	)



	def  A							(  self	:    Tuple	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				_snake_case		=			AutoConfig.from_pretrained(lowercase	)
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase			,  configs=[config]	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_inference_result	)
				self.check_results_dict_not_empty(results.memory_inference_result	)



	def  A							(  self	:    Union[str, Any]	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tinier_bart'
				_snake_case		=			AutoConfig.from_pretrained(lowercase	)
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase			,  configs=[config]	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_inference_result	)
				self.check_results_dict_not_empty(results.memory_inference_result	)



	def  A							(  self	:    Dict	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				_snake_case		=			AutoConfig.from_pretrained(lowercase	)
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase			,  configs=[config]	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_train_result	)
				self.check_results_dict_not_empty(results.memory_train_result	)



	def  A							(  self	:    Dict	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tinier_bart'
				_snake_case		=			AutoConfig.from_pretrained(lowercase	)
				_snake_case		=			PyTorchBenchmarkArguments(
				    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  multi_process=lowercase			,  )
				_snake_case		=			PyTorchBenchmark(lowercase			,  configs=[config]	)
				_snake_case		=			benchmark.run()
				self.check_results_dict_not_empty(results.time_train_result	)
				self.check_results_dict_not_empty(results.memory_train_result	)



	def  A							(  self	:    Optional[Any]	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'
				with tempfile.TemporaryDirectory() as tmp_dir:
							_snake_case		=			PyTorchBenchmarkArguments(
							    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  save_to_csv=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  inference_time_csv_file=os.path.join(lowercase			,  'inf_time.csv'	)			,  train_memory_csv_file=os.path.join(lowercase			,  'train_mem.csv'	)			,  inference_memory_csv_file=os.path.join(lowercase			,  'inf_mem.csv'	)			,  train_time_csv_file=os.path.join(lowercase			,  'train_time.csv'	)			,  env_info_csv_file=os.path.join(lowercase			,  'env.csv'	)			,  multi_process=lowercase			,  )
							_snake_case		=			PyTorchBenchmark(lowercase	)
							benchmark.run()
							self.assertTrue(Path(os.path.join(lowercase			,  'inf_time.csv'	)	).exists()	)
							self.assertTrue(Path(os.path.join(lowercase			,  'train_time.csv'	)	).exists()	)
							self.assertTrue(Path(os.path.join(lowercase			,  'inf_mem.csv'	)	).exists()	)
							self.assertTrue(Path(os.path.join(lowercase			,  'train_mem.csv'	)	).exists()	)
							self.assertTrue(Path(os.path.join(lowercase			,  'env.csv'	)	).exists()	)





	def  A							(  self	:    Union[str, Any]	):


				'''simple docstring'''


				_snake_case		=			'sshleifer/tiny-gpt2'

				def _check_summary_is_not_empty(lowercase	:    Optional[Any]	):
							self.assertTrue(hasattr(lowercase			,  'sequential'	)	)
							self.assertTrue(hasattr(lowercase			,  'cumulative'	)	)
							self.assertTrue(hasattr(lowercase			,  'current'	)	)
							self.assertTrue(hasattr(lowercase			,  'total'	)	)

				with tempfile.TemporaryDirectory() as tmp_dir:
							_snake_case		=			PyTorchBenchmarkArguments(
							    models=[MODEL_ID]			,  training=lowercase			,  inference=lowercase			,  sequence_lengths=[8]			,  batch_sizes=[1]			,  log_filename=os.path.join(lowercase			,  'log.txt'	)			,  log_print=lowercase			,  trace_memory_line_by_line=lowercase			,  multi_process=lowercase			,  )
							_snake_case		=			PyTorchBenchmark(lowercase	)
							_snake_case		=			benchmark.run()
							_check_summary_is_not_empty(result.inference_summary	)
							_check_summary_is_not_empty(result.train_summary	)
							self.assertTrue(Path(os.path.join(lowercase			,  'log.txt'	)	).exists()	)