





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


snake_case__					:							Optional[Any]													=     logging.getLogger(__name__)



def _a				(	)	->						int:







      '''simple docstring'''






      __A          =				argparse.ArgumentParser(
          description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.''' )
      parser.add_argument(
          '''--dataset_name'''					,				type=lowerCamelCase					,				default='''wikitext'''					,				help='''Name of the training. Explore datasets at: hf.co/datasets.'''					,				)
      parser.add_argument(
          '''--dataset_config'''					,				type=lowerCamelCase					,				default='''wikitext-103-raw-v1'''					,				help='''Configuration name of the dataset.''' )
      parser.add_argument(
          '''--tokenizer_name_or_path'''					,				type=lowerCamelCase					,				default='''sayakpaul/unigram-tokenizer-wikitext'''					,				help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''					,				)
      parser.add_argument(
          '''--shard_size'''					,				type=lowerCamelCase					,				default=10_00					,				help='''Number of entries to go in a single shard.'''					,				)
      parser.add_argument('''--split'''					,				type=lowerCamelCase					,				default='''train'''					,				choices=['''train''', '''test''', '''validation'''] )
      parser.add_argument(
          '''--limit'''					,				default=lowerCamelCase					,				type=lowerCamelCase					,				help='''Limit the number of shards (used for debugging).'''					,				)
      parser.add_argument(
          '''--max_length'''					,				type=lowerCamelCase					,				default=5_12					,				help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
          ''' sequence length that is a multiple of 8.'''					,				)
      parser.add_argument(
          '''--output_dir'''					,				default='''tf-tpu'''					,				type=lowerCamelCase					,				help='''Output directory where the TFRecord shards will be saved. If the'''
          ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
          ''' shards will be directly saved to a Google Cloud Storage bucket.'''					,				)

      __A          =				parser.parse_args()
      return args



def _a				(	lowerCamelCase:   Dict )	->						Optional[int]:







      '''simple docstring'''






      def fn(lowerCamelCase:   List[Any] ):
            return tokenizer(examples['''text'''] )

      return fn



def _a				(	lowerCamelCase:   int )	->						Tuple:







      '''simple docstring'''






      __A          =				[]
      for i in range(len(tokenized_data['''input_ids'''] ) ):
            __A          =				{
                '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i] ) ),
                '''attention_mask''': tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i] ) ),
            }
            __A          =				tf.train.Features(feature=lowerCamelCase )
            __A          =				tf.train.Example(features=lowerCamelCase )
            __A          =				example.SerializeToString()
            records.append(lowerCamelCase )
      return records



def _a				(	lowerCamelCase:   Dict )	->						List[str]:







      '''simple docstring'''






      __A          =				datasets.load_dataset(args.dataset_name					,				args.dataset_config					,				split=args.split )

      if args.limit is not None:
            __A          =				min(len(lowerCamelCase )					,				args.limit )
            __A          =				dataset.select(range(lowerCamelCase ) )
            print(F"""Limiting the dataset to {args.limit} entries.""" )

      __A          =				AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir ):
                  os.makedirs(args.output_dir )
            __A          =				os.path.join(args.output_dir					,				args.split )
            if not os.path.exists(lowerCamelCase ):
                  os.makedirs(lowerCamelCase )
      else:
            __A          =				os.path.join(args.output_dir					,				args.split )

      # Tokenize the whole dataset at once.
      __A          =				tokenize_function(lowerCamelCase )
      __A          =				dataset.map(lowerCamelCase					,				batched=lowerCamelCase					,				num_proc=4					,				remove_columns=['''text'''] )

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(lowerCamelCase:   Tuple ):
            # Concatenate all texts.
            __A          =				{k: sum(examples[k]					,				[] ) for k in examples.keys()}
            __A          =				len(concatenated_examples[list(examples.keys() )[0]] )
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            __A          =				(total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            __A          =				{
                k: [t[i : i + args.max_length] for i in range(0					,				lowerCamelCase					,				args.max_length )]
                for k, t in concatenated_examples.items()
            }
            return result

      __A          =				dataset_tokenized.map(lowerCamelCase					,				batched=lowerCamelCase					,				batch_size=10_00					,				num_proc=4 )

      __A          =				0
      __A          =				0
      for shard in range(0					,				len(lowerCamelCase )					,				args.shard_size ):
            __A          =				grouped_dataset[shard : shard + args.shard_size]
            __A          =				len(dataset_snapshot['''input_ids'''] )
            __A          =				os.path.join(lowerCamelCase					,				F"""dataset-{shard_count}-{records_containing}.tfrecord""" )
            __A          =				get_serialized_examples(lowerCamelCase )

            with tf.io.TFRecordWriter(lowerCamelCase ) as out_file:
                  for i in range(len(lowerCamelCase ) ):
                        __A          =				serialized_examples[i]
                        out_file.write(lowerCamelCase )
                  print('''Wrote file {} containing {} records'''.format(lowerCamelCase					,				lowerCamelCase ) )

            shard_count += 1
            total_records += records_containing

      with open(F"""split-{args.split}-records-count.txt"""					,				'''w''' ) as f:
            print(F"""Total {args.split} records: {total_records}"""					,				file=lowerCamelCase )


if __name__ == "__main__":
       snake_case__					:							List[Any]													=     parse_args()
       main(args)












from itertools import permutations



def _a				(	lowerCamelCase:   tuple )	->						bool:







      '''simple docstring'''






      if num[3] % 2 != 0:
            return False

      if (num[2] + num[3] + num[4]) % 3 != 0:
            return False

      if num[5] % 5 != 0:
            return False

      __A          =				[7, 11, 13, 17]
      for i, test in enumerate(lowerCamelCase ):
            if (num[i + 4] * 1_00 + num[i + 5] * 10 + num[i + 6]) % test != 0:
                  return False
      return True



def _a				(	lowerCamelCase:   int = 10 )	->						int:







      '''simple docstring'''






      return sum(
          int(''''''.join(map(lowerCamelCase					,				lowerCamelCase ) ) )
          for num in permutations(range(lowerCamelCase ) )
          if is_substring_divisible(lowerCamelCase ) )


if __name__ == "__main__":
       print(f'{solution() = }')





