'''simple docstring'''





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE_:     Union[str, Any]  =logging.getLogger(__name__)





def 	lowerCAmelCase_				(					)    ->      Union[str, Any]:


     '''simple docstring'''




     UpperCAmelCase_           =   argparse.ArgumentParser(
         description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."			)
     parser.add_argument(
         "--dataset_name"					,    type=snake_case_					,    default="wikitext"					,    help="Name of the training. Explore datasets at: hf.co/datasets."					,    )
     parser.add_argument(
         "--dataset_config"					,    type=snake_case_					,    default="wikitext-103-raw-v1"					,    help="Configuration name of the dataset."			)
     parser.add_argument(
         "--tokenizer_name_or_path"					,    type=snake_case_					,    default="sayakpaul/unigram-tokenizer-wikitext"					,    help="Tokenizer identifier. Can be a local filepath or a Hub identifier."					,    )
     parser.add_argument(
         "--shard_size"					,    type=snake_case_					,    default=10_00					,    help="Number of entries to go in a single shard."					,    )
     parser.add_argument("--split"					,    type=snake_case_					,    default="train"					,    choices=["train", "test", "validation"]			)
     parser.add_argument(
         "--limit"					,    default=snake_case_					,    type=snake_case_					,    help="Limit the number of shards (used for debugging)."					,    )
     parser.add_argument(
         "--max_length"					,    type=snake_case_					,    default=5_12					,    help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
         " sequence length that is a multiple of 8."					,    )
     parser.add_argument(
         "--output_dir"					,    default="tf-tpu"					,    type=snake_case_					,    help="Output directory where the TFRecord shards will be saved. If the"
         " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
         " shards will be directly saved to a Google Cloud Storage bucket."					,    )

     UpperCAmelCase_           =   parser.parse_args()
     return args





def 	lowerCAmelCase_				(					snake_case_       : Dict			)    ->      str:


     '''simple docstring'''




     def fn(snake_case_       : Optional[Any]			):
          return tokenizer(examples["text"]			)

     return fn





def 	lowerCAmelCase_				(					snake_case_       : List[str]			)    ->      Union[str, Any]:


     '''simple docstring'''




     UpperCAmelCase_           =   []
     for i in range(len(tokenized_data["input_ids"]			)			):
          UpperCAmelCase_           =   {
              "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]			)			),
              "attention_mask": tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]			)			),
          }
          UpperCAmelCase_           =   tf.train.Features(feature=snake_case_			)
          UpperCAmelCase_           =   tf.train.Example(features=snake_case_			)
          UpperCAmelCase_           =   example.SerializeToString()
          records.append(snake_case_			)
     return records





def 	lowerCAmelCase_				(					snake_case_       : List[str]			)    ->      str:


     '''simple docstring'''




     UpperCAmelCase_           =   datasets.load_dataset(args.dataset_name					,    args.dataset_config					,    split=args.split			)

     if args.limit is not None:
          UpperCAmelCase_           =   min(len(snake_case_			)					,    args.limit			)
          UpperCAmelCase_           =   dataset.select(range(snake_case_			)			)
          print(f"""Limiting the dataset to {args.limit} entries."""			)

     UpperCAmelCase_           =   AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir			):
               os.makedirs(args.output_dir			)
          UpperCAmelCase_           =   os.path.join(args.output_dir					,    args.split			)
          if not os.path.exists(snake_case_			):
               os.makedirs(snake_case_			)
     else:
          UpperCAmelCase_           =   os.path.join(args.output_dir					,    args.split			)

     # Tokenize the whole dataset at once.
     UpperCAmelCase_           =   tokenize_function(snake_case_			)
     UpperCAmelCase_           =   dataset.map(snake_case_					,    batched=snake_case_					,    num_proc=4					,    remove_columns=["text"]			)

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(snake_case_       : str			):
          # Concatenate all texts.
          UpperCAmelCase_           =   {k: sum(examples[k]					,    []			) for k in examples.keys()}
          UpperCAmelCase_           =   len(concatenated_examples[list(examples.keys()			)[0]]			)
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          UpperCAmelCase_           =   (total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          UpperCAmelCase_           =   {
              k: [t[i : i + args.max_length] for i in range(0					,    snake_case_					,    args.max_length			)]
              for k, t in concatenated_examples.items()
          }
          return result

     UpperCAmelCase_           =   dataset_tokenized.map(snake_case_					,    batched=snake_case_					,    batch_size=10_00					,    num_proc=4			)

     UpperCAmelCase_           =   0
     UpperCAmelCase_           =   0
     for shard in range(0					,    len(snake_case_			)					,    args.shard_size			):
          UpperCAmelCase_           =   grouped_dataset[shard : shard + args.shard_size]
          UpperCAmelCase_           =   len(dataset_snapshot["input_ids"]			)
          UpperCAmelCase_           =   os.path.join(snake_case_					,    f"""dataset-{shard_count}-{records_containing}.tfrecord"""			)
          UpperCAmelCase_           =   get_serialized_examples(snake_case_			)

          with tf.io.TFRecordWriter(snake_case_			) as out_file:
               for i in range(len(snake_case_			)			):
                    UpperCAmelCase_           =   serialized_examples[i]
                    out_file.write(snake_case_			)
               print("Wrote file {} containing {} records".format(snake_case_					,    snake_case_			)			)

          shard_count += 1
          total_records += records_containing

     with open(f"""split-{args.split}-records-count.txt"""					,    "w"			) as f:
          print(f"""Total {args.split} records: {total_records}"""					,    file=snake_case_			)


if __name__ == "__main__":
    SCREAMING_SNAKE_CASE_:     List[str]  =parse_args()
    main(args)









from ..utils import DummyObject, requires_backends






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: List[str]					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Dict					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Any					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Optional[int]					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Optional[int]					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Optional[int]					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)


