





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A    :					Union[str, Any] =      logging.getLogger(__name__)







def    __lowerCamelCase					(							)		->				Tuple:






		"""simple docstring"""

		A__        =   argparse.ArgumentParser(
		    description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""    )
		parser.add_argument(
		    """--dataset_name"""    ,     type=__a    ,     default="""wikitext"""    ,     help="""Name of the training. Explore datasets at: hf.co/datasets."""    ,     )
		parser.add_argument(
		    """--dataset_config"""    ,     type=__a    ,     default="""wikitext-103-raw-v1"""    ,     help="""Configuration name of the dataset."""    )
		parser.add_argument(
		    """--tokenizer_name_or_path"""    ,     type=__a    ,     default="""sayakpaul/unigram-tokenizer-wikitext"""    ,     help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""    ,     )
		parser.add_argument(
		    """--shard_size"""    ,     type=__a    ,     default=1_0_0_0    ,     help="""Number of entries to go in a single shard."""    ,     )
		parser.add_argument("""--split"""    ,     type=__a    ,     default="""train"""    ,     choices=["""train""", """test""", """validation"""]    )
		parser.add_argument(
		    """--limit"""    ,     default=__a    ,     type=__a    ,     help="""Limit the number of shards (used for debugging)."""    ,     )
		parser.add_argument(
		    """--max_length"""    ,     type=__a    ,     default=5_1_2    ,     help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
		    """ sequence length that is a multiple of 8."""    ,     )
		parser.add_argument(
		    """--output_dir"""    ,     default="""tf-tpu"""    ,     type=__a    ,     help="""Output directory where the TFRecord shards will be saved. If the"""
		    """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
		    """ shards will be directly saved to a Google Cloud Storage bucket."""    ,     )

		A__        =   parser.parse_args()
		return args







def    __lowerCamelCase					(							__a   :List[str]    )		->				Any:






		"""simple docstring"""

		def fn(__a   :Optional[int]    ):
				return tokenizer(examples["""text"""]    )

		return fn







def    __lowerCamelCase					(							__a   :str    )		->				Any:






		"""simple docstring"""

		A__        =   []
		for i in range(len(tokenized_data["""input_ids"""]    )    ):
				A__        =   {
				    """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]    )    ),
				    """attention_mask""": tf.train.Feature(
				        intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]    )    ),
				}
				A__        =   tf.train.Features(feature=__a    )
				A__        =   tf.train.Example(features=__a    )
				A__        =   example.SerializeToString()
				records.append(__a    )
		return records







def    __lowerCamelCase					(							__a   :str    )		->				Tuple:






		"""simple docstring"""

		A__        =   datasets.load_dataset(args.dataset_name    ,     args.dataset_config    ,     split=args.split    )

		if args.limit is not None:
				A__        =   min(len(__a    )    ,     args.limit    )
				A__        =   dataset.select(range(__a    )    )
				print(F'Limiting the dataset to {args.limit} entries.'    )

		A__        =   AutoTokenizer.from_pretrained(args.tokenizer_name_or_path    )

		# Handle output directory creation.
		# For serializing into a Google Cloud Storage Bucket, one needs to first
		# create a bucket.
		if "gs" not in args.output_dir:
				if not os.path.exists(args.output_dir    ):
						os.makedirs(args.output_dir    )
				A__        =   os.path.join(args.output_dir    ,     args.split    )
				if not os.path.exists(__a    ):
						os.makedirs(__a    )
		else:
				A__        =   os.path.join(args.output_dir    ,     args.split    )

		# Tokenize the whole dataset at once.
		A__        =   tokenize_function(__a    )
		A__        =   dataset.map(__a    ,     batched=__a    ,     num_proc=4    ,     remove_columns=["""text"""]    )

		# We need to concatenate all our texts together, and then split the result
		# into chunks of a fixed size, which we will call block_size. To do this, we
		# will use the map method again, with the option batched=True. When we use batched=True,
		# the function we pass to map() will be passed multiple inputs at once, allowing us
		# to group them into more or fewer examples than we had in the input.
		# This allows us to create our new fixed-length samples. The advantage of this
		# method is that we don't lose a whole lot of content from the dataset compared to the
		# case where we simply tokenize with a pre-defined max_length.

		def group_texts(__a   :str    ):
				# Concatenate all texts.
				A__        =   {k: sum(examples[k]    ,     []    ) for k in examples.keys()}
				A__        =   len(concatenated_examples[list(examples.keys()    )[0]]    )
				# We drop the small remainder, though you could add padding instead if the model supports it
				# In this, as in all things, we advise you to follow your heart 🫀
				A__        =   (total_length // args.max_length) * args.max_length
				# Split by chunks of max_len.
				A__        =   {
				    k: [t[i : i + args.max_length] for i in range(0    ,     __a    ,     args.max_length    )]
				    for k, t in concatenated_examples.items()
				}
				return result

		A__        =   dataset_tokenized.map(__a    ,     batched=__a    ,     batch_size=1_0_0_0    ,     num_proc=4    )

		A__        =   0
		A__        =   0
		for shard in range(0    ,     len(__a    )    ,     args.shard_size    ):
				A__        =   grouped_dataset[shard : shard + args.shard_size]
				A__        =   len(dataset_snapshot["""input_ids"""]    )
				A__        =   os.path.join(__a    ,     F'dataset-{shard_count}-{records_containing}.tfrecord'    )
				A__        =   get_serialized_examples(__a    )

				with tf.io.TFRecordWriter(__a    ) as out_file:
						for i in range(len(__a    )    ):
								A__        =   serialized_examples[i]
								out_file.write(__a    )
						print("""Wrote file {} containing {} records""".format(__a    ,     __a    )    )

				shard_count += 1
				total_records += records_containing

		with open(F'split-{args.split}-records-count.txt'    ,     """w"""    ) as f:
				print(F'Total {args.split} records: {total_records}'    ,     file=__a    )


if __name__ == "__main__":
					A    :					str =      parse_args()
					main(args)










from math import ceil







def    __lowerCamelCase					(							__a   :int = 1_0_0_1    )		->				int:






		"""simple docstring"""

		A__        =   1

		for i in range(1    ,     int(ceil(n / 2.0    )    )    ):
				A__        =   2 * i + 1
				A__        =   2 * i
				A__        =   total + 4 * odd**2 - 6 * even

		return total


if __name__ == "__main__":
					import sys

					if len(sys.argv) == 1:
										print(solution())
					else:
										try:
															A    :					List[str] =      int(sys.argv[1])
															print(solution(n))
										except ValueError:
															print('''Invalid entry - please enter a number''')



