'''simple docstring'''




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__UpperCAmelCase     							=logging.getLogger(__name__)






def __lowerCAmelCase (							)	->    int:
    __lowerCamelCase		=  argparse.ArgumentParser(
        description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''				)
    parser.add_argument(
        '''--dataset_name'''				,  type=UpperCamelCase__				,  default='''wikitext'''				,  help='''Name of the training. Explore datasets at: hf.co/datasets.'''				,  )
    parser.add_argument(
        '''--dataset_config'''				,  type=UpperCamelCase__				,  default='''wikitext-103-raw-v1'''				,  help='''Configuration name of the dataset.'''				)
    parser.add_argument(
        '''--tokenizer_name_or_path'''				,  type=UpperCamelCase__				,  default='''sayakpaul/unigram-tokenizer-wikitext'''				,  help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''				,  )
    parser.add_argument(
        '''--shard_size'''				,  type=UpperCamelCase__				,  default=10_00				,  help='''Number of entries to go in a single shard.'''				,  )
    parser.add_argument('''--split'''				,  type=UpperCamelCase__				,  default='''train'''				,  choices=['''train''', '''test''', '''validation''']				)
    parser.add_argument(
        '''--limit'''				,  default=UpperCamelCase__				,  type=UpperCamelCase__				,  help='''Limit the number of shards (used for debugging).'''				,  )
    parser.add_argument(
        '''--max_length'''				,  type=UpperCamelCase__				,  default=5_12				,  help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
        ''' sequence length that is a multiple of 8.'''				,  )
    parser.add_argument(
        '''--output_dir'''				,  default='''tf-tpu'''				,  type=UpperCamelCase__				,  help='''Output directory where the TFRecord shards will be saved. If the'''
        ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
        ''' shards will be directly saved to a Google Cloud Storage bucket.'''				,  )

    __lowerCamelCase		=  parser.parse_args()
    return args






def __lowerCAmelCase (							UpperCamelCase__				)	->    Tuple:
    def fn(UpperCamelCase__				):
        return tokenizer(examples['''text''']				)

    return fn






def __lowerCAmelCase (							UpperCamelCase__				)	->    List[Any]:
    __lowerCamelCase		=  []
    for i in range(len(tokenized_data['''input_ids''']				)				):
        __lowerCamelCase		=  {
            """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]				)				),
            """attention_mask""": tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]				)				),
        }
        __lowerCamelCase		=  tf.train.Features(feature=UpperCamelCase__				)
        __lowerCamelCase		=  tf.train.Example(features=UpperCamelCase__				)
        __lowerCamelCase		=  example.SerializeToString()
        records.append(UpperCamelCase__				)
    return records






def __lowerCAmelCase (							UpperCamelCase__				)	->    Union[str, Any]:
    __lowerCamelCase		=  datasets.load_dataset(args.dataset_name				,  args.dataset_config				,  split=args.split				)

    if args.limit is not None:
        __lowerCamelCase		=  min(len(UpperCamelCase__				)				,  args.limit				)
        __lowerCamelCase		=  dataset.select(range(UpperCamelCase__				)				)
        print(f"""Limiting the dataset to {args.limit} entries."""				)

    __lowerCamelCase		=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path				)

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir				):
            os.makedirs(args.output_dir				)
        __lowerCamelCase		=  os.path.join(args.output_dir				,  args.split				)
        if not os.path.exists(UpperCamelCase__				):
            os.makedirs(UpperCamelCase__				)
    else:
        __lowerCamelCase		=  os.path.join(args.output_dir				,  args.split				)

    # Tokenize the whole dataset at once.
    __lowerCamelCase		=  tokenize_function(UpperCamelCase__				)
    __lowerCamelCase		=  dataset.map(UpperCamelCase__				,  batched=UpperCamelCase__				,  num_proc=4				,  remove_columns=['''text''']				)

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(UpperCamelCase__				):
        # Concatenate all texts.
        __lowerCamelCase		=  {k: sum(examples[k]				,  []				) for k in examples.keys()}
        __lowerCamelCase		=  len(concatenated_examples[list(examples.keys()				)[0]]				)
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        __lowerCamelCase		=  (total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        __lowerCamelCase		=  {
            k: [t[i : i + args.max_length] for i in range(0				,  UpperCamelCase__				,  args.max_length				)]
            for k, t in concatenated_examples.items()
        }
        return result

    __lowerCamelCase		=  dataset_tokenized.map(UpperCamelCase__				,  batched=UpperCamelCase__				,  batch_size=10_00				,  num_proc=4				)

    __lowerCamelCase		=  0
    __lowerCamelCase		=  0
    for shard in range(0				,  len(UpperCamelCase__				)				,  args.shard_size				):
        __lowerCamelCase		=  grouped_dataset[shard : shard + args.shard_size]
        __lowerCamelCase		=  len(dataset_snapshot['''input_ids''']				)
        __lowerCamelCase		=  os.path.join(UpperCamelCase__				,  f"""dataset-{shard_count}-{records_containing}.tfrecord"""				)
        __lowerCamelCase		=  get_serialized_examples(UpperCamelCase__				)

        with tf.io.TFRecordWriter(UpperCamelCase__				) as out_file:
            for i in range(len(UpperCamelCase__				)				):
                __lowerCamelCase		=  serialized_examples[i]
                out_file.write(UpperCamelCase__				)
            print('''Wrote file {} containing {} records'''.format(UpperCamelCase__				,  UpperCamelCase__				)				)

        shard_count += 1
        total_records += records_containing

    with open(f"""split-{args.split}-records-count.txt"""				,  '''w'''				) as f:
        print(f"""Total {args.split} records: {total_records}"""				,  file=UpperCamelCase__				)


if __name__ == "__main__":
  __UpperCAmelCase     							=parse_args()
  main(args)





"""simple docstring"""
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from argparse import ArgumentParser

from accelerate.commands.config import get_config_parser
from accelerate.commands.env import env_command_parser
from accelerate.commands.launch import launch_command_parser
from accelerate.commands.test import test_command_parser
from accelerate.commands.tpu import tpu_command_parser

def       _snake_case						(						):
 UpperCAmelCase	:					List[str]	=	ArgumentParser("""Accelerate CLI tool"""			,		usage="""accelerate <command> [<args>]"""			,		allow_abbrev=UpperCamelCase						)
 UpperCAmelCase	:					Dict	=	parser.add_subparsers(help="""accelerate command helpers"""						)

 # Register commands
 get_config_parser(subparsers=UpperCamelCase						)
 env_command_parser(subparsers=UpperCamelCase						)
 launch_command_parser(subparsers=UpperCamelCase						)
 tpu_command_parser(subparsers=UpperCamelCase						)
 test_command_parser(subparsers=UpperCamelCase						)

 # Let's go
 UpperCAmelCase	:					Optional[int]	=	parser.parse_args()

 if not hasattr(UpperCamelCase			,		"""func"""						):
  parser.print_help()
  exit(1						)

 # Run
 args.func(UpperCamelCase						)


if __name__ == "__main__":
      main()
