'''simple docstring'''




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__lowercase          =  logging.getLogger(__name__)






def 				snake_case__							(			)    ->	int:

		'''simple docstring'''


		lowerCAmelCase   					=  argparse.ArgumentParser(
		    description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.""" )
		parser.add_argument(
		    """--dataset_name"""      ,   type=_A      ,   default="""wikitext"""      ,   help="""Name of the training. Explore datasets at: hf.co/datasets."""      ,   )
		parser.add_argument(
		    """--dataset_config"""      ,   type=_A      ,   default="""wikitext-103-raw-v1"""      ,   help="""Configuration name of the dataset.""" )
		parser.add_argument(
		    """--tokenizer_name_or_path"""      ,   type=_A      ,   default="""sayakpaul/unigram-tokenizer-wikitext"""      ,   help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""      ,   )
		parser.add_argument(
		    """--shard_size"""      ,   type=_A      ,   default=1000      ,   help="""Number of entries to go in a single shard."""      ,   )
		parser.add_argument("""--split"""      ,   type=_A      ,   default="""train"""      ,   choices=["""train""", """test""", """validation"""] )
		parser.add_argument(
		    """--limit"""      ,   default=_A      ,   type=_A      ,   help="""Limit the number of shards (used for debugging)."""      ,   )
		parser.add_argument(
		    """--max_length"""      ,   type=_A      ,   default=512      ,   help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
		    """ sequence length that is a multiple of 8."""      ,   )
		parser.add_argument(
		    """--output_dir"""      ,   default="""tf-tpu"""      ,   type=_A      ,   help="""Output directory where the TFRecord shards will be saved. If the"""
		    """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
		    """ shards will be directly saved to a Google Cloud Storage bucket."""      ,   )

		lowerCAmelCase   					=  parser.parse_args()
		return args






def 				snake_case__							(			_A:					Any )    ->	Union[str, Any]:

		'''simple docstring'''


		def fn(_A:					str ):
				return tokenizer(examples["""text"""] )

		return fn






def 				snake_case__							(			_A:					Tuple )    ->	Dict:

		'''simple docstring'''


		lowerCAmelCase   					=  []
		for i in range(len(tokenized_data["""input_ids"""] ) ):
				lowerCAmelCase   					=  {
				    """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i] ) ),
				    """attention_mask""": tf.train.Feature(
				        intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i] ) ),
				}
				lowerCAmelCase   					=  tf.train.Features(feature=_A )
				lowerCAmelCase   					=  tf.train.Example(features=_A )
				lowerCAmelCase   					=  example.SerializeToString()
				records.append(_A )
		return records






def 				snake_case__							(			_A:					Dict )    ->	Any:

		'''simple docstring'''


		lowerCAmelCase   					=  datasets.load_dataset(args.dataset_name      ,   args.dataset_config      ,   split=args.split )

		if args.limit is not None:
				lowerCAmelCase   					=  min(len(_A )      ,   args.limit )
				lowerCAmelCase   					=  dataset.select(range(_A ) )
				print(f"Limiting the dataset to {args.limit} entries." )

		lowerCAmelCase   					=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

		# Handle output directory creation.
		# For serializing into a Google Cloud Storage Bucket, one needs to first
		# create a bucket.
		if "gs" not in args.output_dir:
				if not os.path.exists(args.output_dir ):
						os.makedirs(args.output_dir )
				lowerCAmelCase   					=  os.path.join(args.output_dir      ,   args.split )
				if not os.path.exists(_A ):
						os.makedirs(_A )
		else:
				lowerCAmelCase   					=  os.path.join(args.output_dir      ,   args.split )

		# Tokenize the whole dataset at once.
		lowerCAmelCase   					=  tokenize_function(_A )
		lowerCAmelCase   					=  dataset.map(_A      ,   batched=_A      ,   num_proc=4      ,   remove_columns=["""text"""] )

		# We need to concatenate all our texts together, and then split the result
		# into chunks of a fixed size, which we will call block_size. To do this, we
		# will use the map method again, with the option batched=True. When we use batched=True,
		# the function we pass to map() will be passed multiple inputs at once, allowing us
		# to group them into more or fewer examples than we had in the input.
		# This allows us to create our new fixed-length samples. The advantage of this
		# method is that we don't lose a whole lot of content from the dataset compared to the
		# case where we simply tokenize with a pre-defined max_length.

		def group_texts(_A:					Dict ):
				# Concatenate all texts.
				lowerCAmelCase   					=  {k: sum(examples[k]      ,   [] ) for k in examples.keys()}
				lowerCAmelCase   					=  len(concatenated_examples[list(examples.keys() )[0]] )
				# We drop the small remainder, though you could add padding instead if the model supports it
				# In this, as in all things, we advise you to follow your heart 🫀
				lowerCAmelCase   					=  (total_length // args.max_length) * args.max_length
				# Split by chunks of max_len.
				lowerCAmelCase   					=  {
				    k: [t[i : i + args.max_length] for i in range(0      ,   _A      ,   args.max_length )]
				    for k, t in concatenated_examples.items()
				}
				return result

		lowerCAmelCase   					=  dataset_tokenized.map(_A      ,   batched=_A      ,   batch_size=1000      ,   num_proc=4 )

		lowerCAmelCase   					=  0
		lowerCAmelCase   					=  0
		for shard in range(0      ,   len(_A )      ,   args.shard_size ):
				lowerCAmelCase   					=  grouped_dataset[shard : shard + args.shard_size]
				lowerCAmelCase   					=  len(dataset_snapshot["""input_ids"""] )
				lowerCAmelCase   					=  os.path.join(_A      ,   f"dataset-{shard_count}-{records_containing}.tfrecord" )
				lowerCAmelCase   					=  get_serialized_examples(_A )

				with tf.io.TFRecordWriter(_A ) as out_file:
						for i in range(len(_A ) ):
								lowerCAmelCase   					=  serialized_examples[i]
								out_file.write(_A )
						print("""Wrote file {} containing {} records""".format(_A      ,   _A ) )

				shard_count += 1
				total_records += records_containing

		with open(f"split-{args.split}-records-count.txt"      ,   """w""" ) as f:
				print(f"Total {args.split} records: {total_records}"      ,   file=_A )


if __name__ == "__main__":
						__lowercase          =  parse_args()
						main(args)





'''simple docstring'''




from math import sqrt






def 				snake_case__							(			_A:					int = 1000000 )    ->	int:

		'''simple docstring'''


		lowerCAmelCase   					=  0
		lowerCAmelCase   					=  0
		lowerCAmelCase   					=  42

		while num_cuboids <= limit:
				max_cuboid_size += 1
				for sum_shortest_sides in range(2      ,   2 * max_cuboid_size + 1 ):
						if sqrt(sum_shortest_sides**2 + max_cuboid_size**2 ).is_integer():
								num_cuboids += (
								    min(_A      ,   sum_shortest_sides // 2 )
								    - max(1      ,   sum_shortest_sides - max_cuboid_size )
								    + 1
								)

		return max_cuboid_size


if __name__ == "__main__":
						print(f'{solution() = }')




