
import time
from contextlib import contextmanager
from pathlib import Path

import pytest
import requests
from huggingface_hub.hf_api import HfApi, HfFolder


__a  			=					'__DUMMY_TRANSFORMERS_USER__'
__a  			=					'Dummy User'
__a  			=					'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'

__a  			=					'https://hub-ci.huggingface.co'
__a  			=					CI_HUB_ENDPOINT + '/datasets/{repo_id}/resolve/{revision}/{path}'
__a  			=					CI_HUB_ENDPOINT + '/{repo_id}/resolve/{revision}/{filename}'
__a  			=					Path('~/.huggingface/hub_ci_token').expanduser()




@pytest.fixture
def 				a						(							snake_case__:					Dict					):


				'''simple docstring'''
				monkeypatch.setattr(
				    '''huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE'''						,   _UpperCamelCase					)




@pytest.fixture
def 				a						(							snake_case__:					Union[str, Any]					):


				'''simple docstring'''
				monkeypatch.setattr('''datasets.config.HF_ENDPOINT'''						,   _UpperCamelCase					)
				monkeypatch.setattr('''datasets.config.HUB_DATASETS_URL'''						,   _UpperCamelCase					)




@pytest.fixture
def 				a						(							snake_case__:					Optional[int]					):


				'''simple docstring'''
				monkeypatch.setattr('''huggingface_hub.hf_api.HfFolder.path_token'''						,   _UpperCamelCase					)




@pytest.fixture
def 				a						(							snake_case__:					Tuple						,   snake_case__:					Dict					):


				'''simple docstring'''
				HfFolder.save_token(_UpperCamelCase					)
				yield
				HfFolder.delete_token()




@pytest.fixture(scope='''session'''					)
def 				a						(							):


				'''simple docstring'''
				return HfApi(endpoint=_UpperCamelCase					)




@pytest.fixture(scope='''session'''					)
def 				a						(							snake_case__:					Dict					):


				'''simple docstring'''
				lowercase_       =					HfFolder.get_token()
				HfFolder.save_token(_UpperCamelCase					)
				yield CI_HUB_USER_TOKEN
				if previous_token is not None:
								HfFolder.save_token(_UpperCamelCase					)




@pytest.fixture
def 				a						(							snake_case__:					List[Any]					):


				'''simple docstring'''


				def _cleanup_repo(snake_case__:					Optional[int]					):
								hf_api.delete_repo(_UpperCamelCase						,   token=_UpperCamelCase						,   repo_type='''dataset'''					)

				return _cleanup_repo




@pytest.fixture
def 				a						(							snake_case__:					Dict					):


				'''simple docstring'''


				@contextmanager
				def _temporary_repo(snake_case__:					List[str]					):
								try:
												yield repo_id
								finally:
												cleanup_repo(_UpperCamelCase					)

				return _temporary_repo




@pytest.fixture(scope='''session'''					)
def 				a						(							snake_case__:					Union[str, Any]						,   snake_case__:					Optional[Any]						,   snake_case__:					Optional[Any]					):


				'''simple docstring'''
				lowercase_       =					F'''repo_txt_data-{int(time.time() * 1_0e3					)}'''
				lowercase_       =					F'''{CI_HUB_USER}/{repo_name}'''
				hf_api.create_repo(_UpperCamelCase						,   token=_UpperCamelCase						,   repo_type='''dataset'''						,   private=_UpperCamelCase					)
				hf_api.upload_file(
				    token=_UpperCamelCase						,   path_or_fileobj=str(_UpperCamelCase					)						,   path_in_repo='''data/text_data.txt'''						,   repo_id=_UpperCamelCase						,   repo_type='''dataset'''						,   )
				yield repo_id
				try:
								hf_api.delete_repo(_UpperCamelCase						,   token=_UpperCamelCase						,   repo_type='''dataset'''					)
				except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
								pass




@pytest.fixture()
def 				a						(							snake_case__:					int						,   snake_case__:					Tuple						,   snake_case__:					Dict					):


				'''simple docstring'''
				return hf_private_dataset_repo_txt_data_




@pytest.fixture(scope='''session'''					)
def 				a						(							snake_case__:					Tuple						,   snake_case__:					Union[str, Any]						,   snake_case__:					List[Any]					):


				'''simple docstring'''
				lowercase_       =					F'''repo_zipped_txt_data-{int(time.time() * 1_0e3					)}'''
				lowercase_       =					F'''{CI_HUB_USER}/{repo_name}'''
				hf_api.create_repo(_UpperCamelCase						,   token=_UpperCamelCase						,   repo_type='''dataset'''						,   private=_UpperCamelCase					)
				hf_api.upload_file(
				    token=_UpperCamelCase						,   path_or_fileobj=str(_UpperCamelCase					)						,   path_in_repo='''data.zip'''						,   repo_id=_UpperCamelCase						,   repo_type='''dataset'''						,   )
				yield repo_id
				try:
								hf_api.delete_repo(_UpperCamelCase						,   token=_UpperCamelCase						,   repo_type='''dataset'''					)
				except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
								pass




@pytest.fixture()
def 				a						(							snake_case__:					str						,   snake_case__:					Dict						,   snake_case__:					List[str]					):


				'''simple docstring'''
				return hf_private_dataset_repo_zipped_txt_data_




@pytest.fixture(scope='''session'''					)
def 				a						(							snake_case__:					Optional[int]						,   snake_case__:					Dict						,   snake_case__:					Dict					):


				'''simple docstring'''
				lowercase_       =					F'''repo_zipped_img_data-{int(time.time() * 1_0e3					)}'''
				lowercase_       =					F'''{CI_HUB_USER}/{repo_name}'''
				hf_api.create_repo(_UpperCamelCase						,   token=_UpperCamelCase						,   repo_type='''dataset'''						,   private=_UpperCamelCase					)
				hf_api.upload_file(
				    token=_UpperCamelCase						,   path_or_fileobj=str(_UpperCamelCase					)						,   path_in_repo='''data.zip'''						,   repo_id=_UpperCamelCase						,   repo_type='''dataset'''						,   )
				yield repo_id
				try:
								hf_api.delete_repo(_UpperCamelCase						,   token=_UpperCamelCase						,   repo_type='''dataset'''					)
				except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
								pass




@pytest.fixture()
def 				a						(							snake_case__:					Optional[int]						,   snake_case__:					Union[str, Any]						,   snake_case__:					List[str]					):


				'''simple docstring'''
				return hf_private_dataset_repo_zipped_img_data_











import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase_    =				logging.getLogger(__name__)






def      lowerCamelCase_						(				)		->						Optional[Any]:






				"""simple docstring"""


				snake_case_						:    List[str]				=  argparse.ArgumentParser(
				    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.''' )
				parser.add_argument(
				    '''--dataset_name''' ,				type=_UpperCamelCase ,				default='''wikitext''' ,				help='''Name of the training. Explore datasets at: hf.co/datasets.''' ,				)
				parser.add_argument(
				    '''--dataset_config''' ,				type=_UpperCamelCase ,				default='''wikitext-103-raw-v1''' ,				help='''Configuration name of the dataset.''' )
				parser.add_argument(
				    '''--tokenizer_name_or_path''' ,				type=_UpperCamelCase ,				default='''sayakpaul/unigram-tokenizer-wikitext''' ,				help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.''' ,				)
				parser.add_argument(
				    '''--shard_size''' ,				type=_UpperCamelCase ,				default=1_000 ,				help='''Number of entries to go in a single shard.''' ,				)
				parser.add_argument('''--split''' ,				type=_UpperCamelCase ,				default='''train''' ,				choices=['''train''', '''test''', '''validation'''] )
				parser.add_argument(
				    '''--limit''' ,				default=_UpperCamelCase ,				type=_UpperCamelCase ,				help='''Limit the number of shards (used for debugging).''' ,				)
				parser.add_argument(
				    '''--max_length''' ,				type=_UpperCamelCase ,				default=512 ,				help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
				    ''' sequence length that is a multiple of 8.''' ,				)
				parser.add_argument(
				    '''--output_dir''' ,				default='''tf-tpu''' ,				type=_UpperCamelCase ,				help='''Output directory where the TFRecord shards will be saved. If the'''
				    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
				    ''' shards will be directly saved to a Google Cloud Storage bucket.''' ,				)

				snake_case_						:    List[Any]				=  parser.parse_args()
				return args






def      lowerCamelCase_						(				_UpperCamelCase )		->						Tuple:






				"""simple docstring"""


				def fn(_UpperCamelCase ):
								return tokenizer(examples['''text'''] )

				return fn






def      lowerCamelCase_						(				_UpperCamelCase )		->						Union[str, Any]:






				"""simple docstring"""


				snake_case_						:    Any				=  []
				for i in range(len(tokenized_data['''input_ids'''] ) ):
								snake_case_						:    Any				=  {
								    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i] ) ),
								    '''attention_mask''': tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i] ) ),
								}
								snake_case_						:    Optional[int]				=  tf.train.Features(feature=_UpperCamelCase )
								snake_case_						:    Optional[Any]				=  tf.train.Example(features=_UpperCamelCase )
								snake_case_						:    Optional[Any]				=  example.SerializeToString()
								records.append(_UpperCamelCase )
				return records






def      lowerCamelCase_						(				_UpperCamelCase )		->						Optional[int]:






				"""simple docstring"""


				snake_case_						:    int				=  datasets.load_dataset(args.dataset_name ,				args.dataset_config ,				split=args.split )

				if args.limit is not None:
								snake_case_						:    Union[str, Any]				=  min(len(_UpperCamelCase ) ,				args.limit )
								snake_case_						:    int				=  dataset.select(range(_UpperCamelCase ) )
								print(f'''Limiting the dataset to {args.limit} entries.''' )

				snake_case_						:    Dict				=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir ):
												os.makedirs(args.output_dir )
								snake_case_						:    str				=  os.path.join(args.output_dir ,				args.split )
								if not os.path.exists(_UpperCamelCase ):
												os.makedirs(_UpperCamelCase )
				else:
								snake_case_						:    Optional[Any]				=  os.path.join(args.output_dir ,				args.split )

				# Tokenize the whole dataset at once.
				snake_case_						:    Optional[Any]				=  tokenize_function(_UpperCamelCase )
				snake_case_						:    List[Any]				=  dataset.map(_UpperCamelCase ,				batched=_UpperCamelCase ,				num_proc=4 ,				remove_columns=['''text'''] )

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(_UpperCamelCase ):
								# Concatenate all texts.
								snake_case_						:    Tuple				=  {k: sum(examples[k] ,				[] ) for k in examples.keys()}
								snake_case_						:    List[str]				=  len(concatenated_examples[list(examples.keys() )[0]] )
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								snake_case_						:    int				=  (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								snake_case_						:    Union[str, Any]				=  {
								    k: [t[i : i + args.max_length] for i in range(0 ,				_UpperCamelCase ,				args.max_length )]
								    for k, t in concatenated_examples.items()
								}
								return result

				snake_case_						:    int				=  dataset_tokenized.map(_UpperCamelCase ,				batched=_UpperCamelCase ,				batch_size=1_000 ,				num_proc=4 )

				snake_case_						:    str				=  0
				snake_case_						:    Optional[Any]				=  0
				for shard in range(0 ,				len(_UpperCamelCase ) ,				args.shard_size ):
								snake_case_						:    Any				=  grouped_dataset[shard : shard + args.shard_size]
								snake_case_						:    str				=  len(dataset_snapshot['''input_ids'''] )
								snake_case_						:    Union[str, Any]				=  os.path.join(_UpperCamelCase ,				f'''dataset-{shard_count}-{records_containing}.tfrecord''' )
								snake_case_						:    Dict				=  get_serialized_examples(_UpperCamelCase )

								with tf.io.TFRecordWriter(_UpperCamelCase ) as out_file:
												for i in range(len(_UpperCamelCase ) ):
																snake_case_						:    List[str]				=  serialized_examples[i]
																out_file.write(_UpperCamelCase )
												print('''Wrote file {} containing {} records'''.format(_UpperCamelCase ,				_UpperCamelCase ) )

								shard_count += 1
								total_records += records_containing

				with open(f'''split-{args.split}-records-count.txt''' ,				'''w''' ) as f:
								print(f'''Total {args.split} records: {total_records}''' ,				file=_UpperCamelCase )


if __name__ == "__main__":
	lowerCAmelCase_    =				parse_args()
	main(args)
