
"""simple docstring"""






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase_							=					logging.getLogger(__name__)
def 					UpperCamelCase		( )		->Union[str, Any]:

   """simple docstring"""



   a_							=			argparse.ArgumentParser(
       description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."			)
   parser.add_argument(
       "--dataset_name"      ,					type=__lowercase      ,					default="wikitext"      ,					help="Name of the training. Explore datasets at: hf.co/datasets."      ,					)
   parser.add_argument(
       "--dataset_config"      ,					type=__lowercase      ,					default="wikitext-103-raw-v1"      ,					help="Configuration name of the dataset."			)
   parser.add_argument(
       "--tokenizer_name_or_path"      ,					type=__lowercase      ,					default="sayakpaul/unigram-tokenizer-wikitext"      ,					help="Tokenizer identifier. Can be a local filepath or a Hub identifier."      ,					)
   parser.add_argument(
       "--shard_size"      ,					type=__lowercase      ,					default=1_000      ,					help="Number of entries to go in a single shard."      ,					)
   parser.add_argument("--split"      ,					type=__lowercase      ,					default="train"      ,					choices=["train", "test", "validation"]			)
   parser.add_argument(
       "--limit"      ,					default=__lowercase      ,					type=__lowercase      ,					help="Limit the number of shards (used for debugging)."      ,					)
   parser.add_argument(
       "--max_length"      ,					type=__lowercase      ,					default=512      ,					help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
       " sequence length that is a multiple of 8."      ,					)
   parser.add_argument(
       "--output_dir"      ,					default="tf-tpu"      ,					type=__lowercase      ,					help="Output directory where the TFRecord shards will be saved. If the"
       " path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"
       " shards will be directly saved to a Google Cloud Storage bucket."      ,					)

   a_							=			parser.parse_args()
   return args
def 					UpperCamelCase		( UpperCAmelCase			)		->str:

   """simple docstring"""



   def fn(UpperCAmelCase			):
      return tokenizer(examples["text"]			)

   return fn
def 					UpperCamelCase		( UpperCAmelCase			)		->int:

   """simple docstring"""



   a_							=			[]
   for i in range(len(tokenized_data["input_ids"]			)			):
      a_							=			{
          "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]			)			),
          "attention_mask": tf.train.Feature(
              intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]			)			),
      }
      a_							=			tf.train.Features(feature=__lowercase			)
      a_							=			tf.train.Example(features=__lowercase			)
      a_							=			example.SerializeToString()
      records.append(__lowercase			)
   return records
def 					UpperCamelCase		( UpperCAmelCase			)		->Any:

   """simple docstring"""



   a_							=			datasets.load_dataset(args.dataset_name      ,					args.dataset_config      ,					split=args.split			)

   if args.limit is not None:
      a_							=			min(len(__lowercase			)      ,					args.limit			)
      a_							=			dataset.select(range(__lowercase			)			)
      print(F'''Limiting the dataset to {args.limit} entries.'''			)

   a_							=			AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

   # Handle output directory creation.
   # For serializing into a Google Cloud Storage Bucket, one needs to first
   # create a bucket.
   if "gs" not in args.output_dir:
      if not os.path.exists(args.output_dir			):
         os.makedirs(args.output_dir			)
      a_							=			os.path.join(args.output_dir      ,					args.split			)
      if not os.path.exists(__lowercase			):
         os.makedirs(__lowercase			)
   else:
      a_							=			os.path.join(args.output_dir      ,					args.split			)

   # Tokenize the whole dataset at once.
   a_							=			tokenize_function(__lowercase			)
   a_							=			dataset.map(__lowercase      ,					batched=__lowercase      ,					num_proc=4      ,					remove_columns=["text"]			)

   # We need to concatenate all our texts together, and then split the result
   # into chunks of a fixed size, which we will call block_size. To do this, we
   # will use the map method again, with the option batched=True. When we use batched=True,
   # the function we pass to map() will be passed multiple inputs at once, allowing us
   # to group them into more or fewer examples than we had in the input.
   # This allows us to create our new fixed-length samples. The advantage of this
   # method is that we don't lose a whole lot of content from the dataset compared to the
   # case where we simply tokenize with a pre-defined max_length.

   def group_texts(UpperCAmelCase			):
      # Concatenate all texts.
      a_							=			{k: sum(examples[k]      ,					[]			) for k in examples.keys()}
      a_							=			len(concatenated_examples[list(examples.keys()			)[0]]			)
      # We drop the small remainder, though you could add padding instead if the model supports it
      # In this, as in all things, we advise you to follow your heart 🫀
      a_							=			(total_length // args.max_length) * args.max_length
      # Split by chunks of max_len.
      a_							=			{
          k: [t[i : i + args.max_length] for i in range(0      ,					__lowercase      ,					args.max_length			)]
          for k, t in concatenated_examples.items()
      }
      return result

   a_							=			dataset_tokenized.map(__lowercase      ,					batched=__lowercase      ,					batch_size=1_000      ,					num_proc=4			)

   a_							=			0
   a_							=			0
   for shard in range(0      ,					len(__lowercase			)      ,					args.shard_size			):
      a_							=			grouped_dataset[shard : shard + args.shard_size]
      a_							=			len(dataset_snapshot["input_ids"]			)
      a_							=			os.path.join(__lowercase      ,					F'''dataset-{shard_count}-{records_containing}.tfrecord'''			)
      a_							=			get_serialized_examples(__lowercase			)

      with tf.io.TFRecordWriter(__lowercase			) as out_file:
         for i in range(len(__lowercase			)			):
            a_							=			serialized_examples[i]
            out_file.write(__lowercase			)
         print("Wrote file {} containing {} records".format(__lowercase      ,					__lowercase			)			)

      shard_count += 1
      total_records += records_containing

   with open(F'''split-{args.split}-records-count.txt'''      ,					"w"			) as f:
      print(F'''Total {args.split} records: {total_records}'''      ,					file=__lowercase			)


if __name__ == "__main__":
      UpperCamelCase_							=					parse_args()
      main(args)

from __future__ import annotations

import time
from collections.abc import Sequence
from random import randint

from matplotlib import pyplot as plt






def        a_		(       __lowercase   :	Sequence[float]       ,					__lowercase   :	int       ,					__lowercase   :	int ) ->  tuple[int | None, int | None, float]:
			if not arr:
						return None, None, 0
			if low == high:
						return low, high, arr[low]

			_snake_case		=			(low + high) // 2
			_snake_case	,					_snake_case	,					_snake_case		=			max_subarray(__lowercase       ,					__lowercase       ,					__lowercase )
			_snake_case	,					_snake_case	,					_snake_case		=			max_subarray(__lowercase       ,					mid + 1       ,					__lowercase )
			_snake_case	,					_snake_case	,					_snake_case		=			max_cross_sum(__lowercase       ,					__lowercase       ,					__lowercase       ,					__lowercase )
			if left_sum >= right_sum and left_sum >= cross_sum:
						return left_low, left_high, left_sum
			elif right_sum >= left_sum and right_sum >= cross_sum:
						return right_low, right_high, right_sum
			return cross_left, cross_right, cross_sum






def        a_		(       __lowercase   :	Sequence[float]       ,					__lowercase   :	int       ,					__lowercase   :	int       ,					__lowercase   :	int ) ->  tuple[int, int, float]:
			_snake_case	,					_snake_case		=			float('-inf' ), -1
			_snake_case	,					_snake_case		=			float('-inf' ), -1

			_snake_case		=			0
			for i in range(__lowercase       ,					low - 1       ,					-1 ):
						summ += arr[i]
						if summ > left_sum:
									_snake_case		=			summ
									_snake_case		=			i

			_snake_case		=			0
			for i in range(mid + 1       ,					high + 1 ):
						summ += arr[i]
						if summ > right_sum:
									_snake_case		=			summ
									_snake_case		=			i

			return max_left, max_right, (left_sum + right_sum)






def        a_		(       __lowercase   :	int ) ->  float:
			_snake_case		=			[randint(1       ,					__lowercase ) for _ in range(__lowercase )]
			_snake_case		=			time.time()
			max_subarray(__lowercase       ,					0       ,					input_size - 1 )
			_snake_case		=			time.time()
			return end - start






def        a_		(       ) ->  None:
			_snake_case		=			[10, 100, 1_000, 10_000, 50_000, 100_000, 200_000, 300_000, 400_000, 500_000]
			_snake_case		=			[time_max_subarray(__lowercase ) for input_size in input_sizes]
			print('No of Inputs\t\tTime Taken' )
			for input_size, runtime in zip(__lowercase       ,					__lowercase ):
						print(__lowercase       ,					'\t\t'       ,					__lowercase )
			plt.plot(__lowercase       ,					__lowercase )
			plt.xlabel('Number of Inputs' )
			plt.ylabel('Time taken in seconds' )
			plt.show()


if __name__ == "__main__":
			from doctest import testmod

			testmod()