



"""simple docstring"""
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A:   str								=		logging.getLogger(__name__)

def       _snake_case						(						):
 UpperCAmelCase	:					List[str]	=	argparse.ArgumentParser(
     description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""						)
 parser.add_argument(
     """--dataset_name"""			,		type=UpperCamelCase			,		default="""wikitext"""			,		help="""Name of the training. Explore datasets at: hf.co/datasets."""			,		)
 parser.add_argument(
     """--dataset_config"""			,		type=UpperCamelCase			,		default="""wikitext-103-raw-v1"""			,		help="""Configuration name of the dataset."""						)
 parser.add_argument(
     """--tokenizer_name_or_path"""			,		type=UpperCamelCase			,		default="""sayakpaul/unigram-tokenizer-wikitext"""			,		help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""			,		)
 parser.add_argument(
     """--shard_size"""			,		type=UpperCamelCase			,		default=1000			,		help="""Number of entries to go in a single shard."""			,		)
 parser.add_argument("""--split"""			,		type=UpperCamelCase			,		default="""train"""			,		choices=["""train""", """test""", """validation"""]						)
 parser.add_argument(
     """--limit"""			,		default=UpperCamelCase			,		type=UpperCamelCase			,		help="""Limit the number of shards (used for debugging)."""			,		)
 parser.add_argument(
     """--max_length"""			,		type=UpperCamelCase			,		default=512			,		help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
     """ sequence length that is a multiple of 8."""			,		)
 parser.add_argument(
     """--output_dir"""			,		default="""tf-tpu"""			,		type=UpperCamelCase			,		help="""Output directory where the TFRecord shards will be saved. If the"""
     """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
     """ shards will be directly saved to a Google Cloud Storage bucket."""			,		)

 UpperCAmelCase	:					Dict	=	parser.parse_args()
 return args

def       _snake_case						(						UpperCamelCase						:		str						):
 def fn(UpperCamelCase						:		Optional[int]						):
  return tokenizer(examples["""text"""]						)

 return fn

def       _snake_case						(						UpperCamelCase						:		Optional[int]						):
 UpperCAmelCase	:					List[Any]	=	[]
 for i in range(len(tokenized_data["""input_ids"""]						)						):
  UpperCAmelCase	:					List[str]	=	{
      """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]						)						),
      """attention_mask""": tf.train.Feature(
          intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]						)						),
  }
  UpperCAmelCase	:					Union[str, Any]	=	tf.train.Features(feature=UpperCamelCase						)
  UpperCAmelCase	:					int	=	tf.train.Example(features=UpperCamelCase						)
  UpperCAmelCase	:					Tuple	=	example.SerializeToString()
  records.append(UpperCamelCase						)
 return records

def       _snake_case						(						UpperCamelCase						:		List[Any]						):
 UpperCAmelCase	:					Any	=	datasets.load_dataset(args.dataset_name			,		args.dataset_config			,		split=args.split						)

 if args.limit is not None:
  UpperCAmelCase	:					List[Any]	=	min(len(UpperCamelCase						)			,		args.limit						)
  UpperCAmelCase	:					Any	=	dataset.select(range(UpperCamelCase						)						)
  print(F"Limiting the dataset to {args.limit} entries."						)

 UpperCAmelCase	:					Optional[int]	=	AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

 # Handle output directory creation.
 # For serializing into a Google Cloud Storage Bucket, one needs to first
 # create a bucket.
 if "gs" not in args.output_dir:
  if not os.path.exists(args.output_dir						):
   os.makedirs(args.output_dir						)
  UpperCAmelCase	:					Optional[Any]	=	os.path.join(args.output_dir			,		args.split						)
  if not os.path.exists(UpperCamelCase						):
   os.makedirs(UpperCamelCase						)
 else:
  UpperCAmelCase	:					Tuple	=	os.path.join(args.output_dir			,		args.split						)

 # Tokenize the whole dataset at once.
 UpperCAmelCase	:					str	=	tokenize_function(UpperCamelCase						)
 UpperCAmelCase	:					str	=	dataset.map(UpperCamelCase			,		batched=UpperCamelCase			,		num_proc=4			,		remove_columns=["""text"""]						)

 # We need to concatenate all our texts together, and then split the result
 # into chunks of a fixed size, which we will call block_size. To do this, we
 # will use the map method again, with the option batched=True. When we use batched=True,
 # the function we pass to map() will be passed multiple inputs at once, allowing us
 # to group them into more or fewer examples than we had in the input.
 # This allows us to create our new fixed-length samples. The advantage of this
 # method is that we don't lose a whole lot of content from the dataset compared to the
 # case where we simply tokenize with a pre-defined max_length.

 def group_texts(UpperCamelCase						:		Any						):
  # Concatenate all texts.
  UpperCAmelCase	:					List[str]	=	{k: sum(examples[k]			,		[]						) for k in examples.keys()}
  UpperCAmelCase	:					str	=	len(concatenated_examples[list(examples.keys()						)[0]]						)
  # We drop the small remainder, though you could add padding instead if the model supports it
  # In this, as in all things, we advise you to follow your heart 🫀
  UpperCAmelCase	:					Dict	=	(total_length // args.max_length) * args.max_length
  # Split by chunks of max_len.
  UpperCAmelCase	:					Any	=	{
      k: [t[i : i + args.max_length] for i in range(0			,		UpperCamelCase			,		args.max_length						)]
      for k, t in concatenated_examples.items()
  }
  return result

 UpperCAmelCase	:					Any	=	dataset_tokenized.map(UpperCamelCase			,		batched=UpperCamelCase			,		batch_size=1000			,		num_proc=4						)

 UpperCAmelCase	:					int	=	0
 UpperCAmelCase	:					Optional[Any]	=	0
 for shard in range(0			,		len(UpperCamelCase						)			,		args.shard_size						):
  UpperCAmelCase	:					Optional[Any]	=	grouped_dataset[shard : shard + args.shard_size]
  UpperCAmelCase	:					Dict	=	len(dataset_snapshot["""input_ids"""]						)
  UpperCAmelCase	:					Dict	=	os.path.join(UpperCamelCase			,		F"dataset-{shard_count}-{records_containing}.tfrecord"						)
  UpperCAmelCase	:					Union[str, Any]	=	get_serialized_examples(UpperCamelCase						)

  with tf.io.TFRecordWriter(UpperCamelCase						) as out_file:
   for i in range(len(UpperCamelCase						)						):
    UpperCAmelCase	:					List[Any]	=	serialized_examples[i]
    out_file.write(UpperCamelCase						)
   print("""Wrote file {} containing {} records""".format(UpperCamelCase			,		UpperCamelCase						)						)

  shard_count += 1
  total_records += records_containing

 with open(F"split-{args.split}-records-count.txt"			,		"""w"""						) as f:
  print(F"Total {args.split} records: {total_records}"			,		file=UpperCamelCase						)


if __name__ == "__main__":
      A:   Union[str, Any]								=		parse_args()
      main(args)





"""simple docstring"""
from math import pi, sqrt, tan

def       _snake_case						(						UpperCamelCase						:		float						):
 if side_length < 0:
  raise ValueError("""surface_area_cube() only accepts non-negative values"""						)
 return 6 * side_length**2

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if length < 0 or breadth < 0 or height < 0:
  raise ValueError("""surface_area_cuboid() only accepts non-negative values"""						)
 return 2 * ((length * breadth) + (breadth * height) + (length * height))

def       _snake_case						(						UpperCamelCase						:		float						):
 if radius < 0:
  raise ValueError("""surface_area_sphere() only accepts non-negative values"""						)
 return 4 * pi * radius**2

def       _snake_case						(						UpperCamelCase						:		float						):
 if radius < 0:
  raise ValueError("""surface_area_hemisphere() only accepts non-negative values"""						)
 return 3 * pi * radius**2

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if radius < 0 or height < 0:
  raise ValueError("""surface_area_cone() only accepts non-negative values"""						)
 return pi * radius * (radius + (height**2 + radius**2) ** 0.5)

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if radius_a < 0 or radius_a < 0 or height < 0:
  raise ValueError(
      """surface_area_conical_frustum() only accepts non-negative values"""						)
 UpperCAmelCase	:					Tuple	=	(height**2 + (radius_a - radius_a) ** 2) ** 0.5
 return pi * ((slant_height * (radius_a + radius_a)) + radius_a**2 + radius_a**2)

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if radius < 0 or height < 0:
  raise ValueError("""surface_area_cylinder() only accepts non-negative values"""						)
 return 2 * pi * radius * (height + radius)

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if torus_radius < 0 or tube_radius < 0:
  raise ValueError("""surface_area_torus() only accepts non-negative values"""						)
 if torus_radius < tube_radius:
  raise ValueError(
      """surface_area_torus() does not support spindle or self intersecting tori"""						)
 return 4 * pow(UpperCamelCase			,		2						) * torus_radius * tube_radius

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if length < 0 or width < 0:
  raise ValueError("""area_rectangle() only accepts non-negative values"""						)
 return length * width

def       _snake_case						(						UpperCamelCase						:		float						):
 if side_length < 0:
  raise ValueError("""area_square() only accepts non-negative values"""						)
 return side_length**2

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if base < 0 or height < 0:
  raise ValueError("""area_triangle() only accepts non-negative values"""						)
 return (base * height) / 2

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if sidea < 0 or sidea < 0 or sidea < 0:
  raise ValueError("""area_triangle_three_sides() only accepts non-negative values"""						)
 elif sidea + sidea < sidea or sidea + sidea < sidea or sidea + sidea < sidea:
  raise ValueError("""Given three sides do not form a triangle"""						)
 UpperCAmelCase	:					Union[str, Any]	=	(sidea + sidea + sidea) / 2
 UpperCAmelCase	:					Union[str, Any]	=	sqrt(
     semi_perimeter
     * (semi_perimeter - sidea)
     * (semi_perimeter - sidea)
     * (semi_perimeter - sidea)						)
 return area

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if base < 0 or height < 0:
  raise ValueError("""area_parallelogram() only accepts non-negative values"""						)
 return base * height

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if basea < 0 or basea < 0 or height < 0:
  raise ValueError("""area_trapezium() only accepts non-negative values"""						)
 return 1 / 2 * (basea + basea) * height

def       _snake_case						(						UpperCamelCase						:		float						):
 if radius < 0:
  raise ValueError("""area_circle() only accepts non-negative values"""						)
 return pi * radius**2

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if radius_x < 0 or radius_y < 0:
  raise ValueError("""area_ellipse() only accepts non-negative values"""						)
 return pi * radius_x * radius_y

def       _snake_case						(						UpperCamelCase						:		float			,		UpperCamelCase						:		float						):
 if diagonal_a < 0 or diagonal_a < 0:
  raise ValueError("""area_rhombus() only accepts non-negative values"""						)
 return 1 / 2 * diagonal_a * diagonal_a

def       _snake_case						(						UpperCamelCase						:		int			,		UpperCamelCase						:		float						):
 if not isinstance(UpperCamelCase			,		UpperCamelCase						) or sides < 3:
  raise ValueError(
      """area_reg_polygon() only accepts integers greater than or \
equal to three as number of sides"""						)
 elif length < 0:
  raise ValueError(
      """area_reg_polygon() only accepts non-negative values as \
length of a side"""						)
 return (sides * length**2) / (4 * tan(pi / sides						))
 return (sides * length**2) / (4 * tan(pi / sides						))


if __name__ == "__main__":
      import doctest

      doctest.testmod(verbose=True)  # verbose so we can see methods missing tests

      print("[DEMO] Areas of various geometric shapes: \n")
      print(f"""Rectangle: {area_rectangle(1_0, 2_0) = }""")
      print(f"""Square: {area_square(1_0) = }""")
      print(f"""Triangle: {area_triangle(1_0, 1_0) = }""")
      print(f"""Triangle: {area_triangle_three_sides(5, 1_2, 1_3) = }""")
      print(f"""Parallelogram: {area_parallelogram(1_0, 2_0) = }""")
      print(f"""Rhombus: {area_rhombus(1_0, 2_0) = }""")
      print(f"""Trapezium: {area_trapezium(1_0, 2_0, 3_0) = }""")
      print(f"""Circle: {area_circle(2_0) = }""")
      print(f"""Ellipse: {area_ellipse(1_0, 2_0) = }""")
      print("\nSurface Areas of various geometric shapes: \n")
      print(f"""Cube: {surface_area_cube(2_0) = }""")
      print(f"""Cuboid: {surface_area_cuboid(1_0, 2_0, 3_0) = }""")
      print(f"""Sphere: {surface_area_sphere(2_0) = }""")
      print(f"""Hemisphere: {surface_area_hemisphere(2_0) = }""")
      print(f"""Cone: {surface_area_cone(1_0, 2_0) = }""")
      print(f"""Conical Frustum: {surface_area_conical_frustum(1_0, 2_0, 3_0) = }""")
      print(f"""Cylinder: {surface_area_cylinder(1_0, 2_0) = }""")
      print(f"""Torus: {surface_area_torus(2_0, 1_0) = }""")
      print(f"""Equilateral Triangle: {area_reg_polygon(3, 1_0) = }""")
      print(f"""Square: {area_reg_polygon(4, 1_0) = }""")
      print(f"""Reqular Pentagon: {area_reg_polygon(5, 1_0) = }""")
