import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


a  :      Optional[int]            =    logging.getLogger(__name__)
def 		snake_case__	(		):
   lowerCAmelCase_:							Optional[int]       =	argparse.ArgumentParser(
       description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."					)
   parser.add_argument(
       "--dataset_name"							,				type=lowercase							,				default="wikitext"							,				help="Name of the training. Explore datasets at: hf.co/datasets."							,				)
   parser.add_argument(
       "--dataset_config"							,				type=lowercase							,				default="wikitext-103-raw-v1"							,				help="Configuration name of the dataset."					)
   parser.add_argument(
       "--tokenizer_name_or_path"							,				type=lowercase							,				default="sayakpaul/unigram-tokenizer-wikitext"							,				help="Tokenizer identifier. Can be a local filepath or a Hub identifier."							,				)
   parser.add_argument(
       "--shard_size"							,				type=lowercase							,				default=1000							,				help="Number of entries to go in a single shard."							,				)
   parser.add_argument("--split"							,				type=lowercase							,				default="train"							,				choices=["train", "test", "validation"]					)
   parser.add_argument(
       "--limit"							,				default=lowercase							,				type=lowercase							,				help="Limit the number of shards (used for debugging)."							,				)
   parser.add_argument(
       "--max_length"							,				type=lowercase							,				default=512							,				help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
       " sequence length that is a multiple of 8."							,				)
   parser.add_argument(
       "--output_dir"							,				default="tf-tpu"							,				type=lowercase							,				help="Output directory where the TFRecord shards will be saved. If the"
       " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
       " shards will be directly saved to a Google Cloud Storage bucket."							,				)

   lowerCAmelCase_:							Union[str, Any]       =	parser.parse_args()
   return args
def 		snake_case__	(		lowercase					):
   def fn(lowercase					):
      return tokenizer(examples["text"]					)

   return fn
def 		snake_case__	(		lowercase					):
   lowerCAmelCase_:							Dict       =	[]
   for i in range(len(tokenized_data["input_ids"]					)					):
      lowerCAmelCase_:							List[Any]       =	{
          "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]					)					),
          "attention_mask": tf.train.Feature(
              intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]					)					),
      }
      lowerCAmelCase_:							str       =	tf.train.Features(feature=lowercase					)
      lowerCAmelCase_:							Optional[int]       =	tf.train.Example(features=lowercase					)
      lowerCAmelCase_:							int       =	example.SerializeToString()
      records.append(lowercase					)
   return records
def 		snake_case__	(		lowercase					):
   lowerCAmelCase_:							List[str]       =	datasets.load_dataset(args.dataset_name							,				args.dataset_config							,				split=args.split					)

   if args.limit is not None:
      lowerCAmelCase_:							Dict       =	min(len(lowercase					)							,				args.limit					)
      lowerCAmelCase_:							List[str]       =	dataset.select(range(lowercase					)					)
      print(F'''Limiting the dataset to {args.limit} entries.'''					)

   lowerCAmelCase_:							Dict       =	AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

   # Handle output directory creation.
   # For serializing into a Google Cloud Storage Bucket, one needs to first
   # create a bucket.
   if "gs" not in args.output_dir:
      if not os.path.exists(args.output_dir					):
         os.makedirs(args.output_dir					)
      lowerCAmelCase_:							Optional[int]       =	os.path.join(args.output_dir							,				args.split					)
      if not os.path.exists(lowercase					):
         os.makedirs(lowercase					)
   else:
      lowerCAmelCase_:							Any       =	os.path.join(args.output_dir							,				args.split					)

   # Tokenize the whole dataset at once.
   lowerCAmelCase_:							List[str]       =	tokenize_function(lowercase					)
   lowerCAmelCase_:							Any       =	dataset.map(lowercase							,				batched=lowercase							,				num_proc=4							,				remove_columns=["text"]					)

   # We need to concatenate all our texts together, and then split the result
   # into chunks of a fixed size, which we will call block_size. To do this, we
   # will use the map method again, with the option batched=True. When we use batched=True,
   # the function we pass to map() will be passed multiple inputs at once, allowing us
   # to group them into more or fewer examples than we had in the input.
   # This allows us to create our new fixed-length samples. The advantage of this
   # method is that we don't lose a whole lot of content from the dataset compared to the
   # case where we simply tokenize with a pre-defined max_length.

   def group_texts(lowercase					):
      # Concatenate all texts.
      lowerCAmelCase_:							int       =	{k: sum(examples[k]							,				[]					) for k in examples.keys()}
      lowerCAmelCase_:							Optional[Any]       =	len(concatenated_examples[list(examples.keys()					)[0]]					)
      # We drop the small remainder, though you could add padding instead if the model supports it
      # In this, as in all things, we advise you to follow your heart 🫀
      lowerCAmelCase_:							List[str]       =	(total_length // args.max_length) * args.max_length
      # Split by chunks of max_len.
      lowerCAmelCase_:							List[str]       =	{
          k: [t[i : i + args.max_length] for i in range(0							,				lowercase							,				args.max_length					)]
          for k, t in concatenated_examples.items()
      }
      return result

   lowerCAmelCase_:							str       =	dataset_tokenized.map(lowercase							,				batched=lowercase							,				batch_size=1000							,				num_proc=4					)

   lowerCAmelCase_:							List[str]       =	0
   lowerCAmelCase_:							Optional[Any]       =	0
   for shard in range(0							,				len(lowercase					)							,				args.shard_size					):
      lowerCAmelCase_:							Optional[int]       =	grouped_dataset[shard : shard + args.shard_size]
      lowerCAmelCase_:							Optional[int]       =	len(dataset_snapshot["input_ids"]					)
      lowerCAmelCase_:							Dict       =	os.path.join(lowercase							,				F'''dataset-{shard_count}-{records_containing}.tfrecord'''					)
      lowerCAmelCase_:							List[Any]       =	get_serialized_examples(lowercase					)

      with tf.io.TFRecordWriter(lowercase					) as out_file:
         for i in range(len(lowercase					)					):
            lowerCAmelCase_:							List[str]       =	serialized_examples[i]
            out_file.write(lowercase					)
         print("Wrote file {} containing {} records".format(lowercase							,				lowercase					)					)

      shard_count += 1
      total_records += records_containing

   with open(F'''split-{args.split}-records-count.txt'''							,				"w"					) as f:
      print(F'''Total {args.split} records: {total_records}'''							,				file=lowercase					)


if __name__ == "__main__":
 a  :      Optional[int]            =    parse_args()
 main(args)
import contextlib
import os
import sqlitea

import pytest

from datasets import Dataset, Features, Value
from datasets.io.sql import SqlDatasetReader, SqlDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases, require_sqlalchemy
def 		snake_case__	(		lowercase							,				lowercase					):
   assert isinstance(lowercase							,				lowercase					)
   assert dataset.num_rows == 4
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_1", "col_2", "col_3"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@require_sqlalchemy
@pytest.mark.parametrize("keep_in_memory"							,				[False, True]					)
def 		snake_case__	(		lowercase							,				lowercase							,				lowercase							,				lowercase					):
   lowerCAmelCase_:							Any       =	tmp_path / "cache"
   lowerCAmelCase_:							int       =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      lowerCAmelCase_:							Optional[int]       =	SqlDatasetReader(
          "dataset"							,				"sqlite:///" + sqlite_path							,				cache_dir=lowercase							,				keep_in_memory=lowercase					).read()
   _check_sql_dataset(lowercase							,				lowercase					)
@require_sqlalchemy
@pytest.mark.parametrize(
    "features"							,				[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]							,				)
def 		snake_case__	(		lowercase							,				lowercase							,				lowercase							,				lowercase					):
   lowerCAmelCase_:							List[str]       =	tmp_path / "cache"
   lowerCAmelCase_:							int       =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
   lowerCAmelCase_:							List[str]       =	features.copy() if features else default_expected_features
   lowerCAmelCase_:							Any       =	(
       Features({feature: Value(lowercase					) for feature, dtype in features.items()}					) if features is not None else None
   )
   lowerCAmelCase_:							int       =	SqlDatasetReader("dataset"							,				"sqlite:///" + sqlite_path							,				features=lowercase							,				cache_dir=lowercase					).read()
   _check_sql_dataset(lowercase							,				lowercase					)
def 		snake_case__	(		lowercase					):
   with contextlib.closing(sqlitea.connect(lowercase					)					) as con:
      lowerCAmelCase_:							Any       =	con.cursor()
      cur.execute("SELECT * FROM dataset"					)
      for row in cur:
         yield row
@require_sqlalchemy
def 		snake_case__	(		lowercase							,				lowercase							,				lowercase					):
   lowerCAmelCase_:							Optional[int]       =	tmp_path / "cache"
   lowerCAmelCase_:							Optional[Any]       =	os.path.join(lowercase							,				"tmp.sql"					)
   lowerCAmelCase_:							str       =	SqlDatasetReader("dataset"							,				"sqlite:///" + sqlite_path							,				cache_dir=lowercase					).read()
   SqlDatasetWriter(lowercase							,				"dataset"							,				"sqlite:///" + output_sqlite_path							,				num_proc=1					).write()

   lowerCAmelCase_:							Union[str, Any]       =	iter_sql_file(lowercase					)
   lowerCAmelCase_:							str       =	iter_sql_file(lowercase					)

   for rowa, rowa in zip(lowercase							,				lowercase					):
      assert rowa == rowa
@require_sqlalchemy
def 		snake_case__	(		lowercase							,				lowercase							,				lowercase					):
   lowerCAmelCase_:							str       =	tmp_path / "cache"
   lowerCAmelCase_:							Optional[int]       =	os.path.join(lowercase							,				"tmp.sql"					)
   lowerCAmelCase_:							Optional[Any]       =	SqlDatasetReader("dataset"							,				"sqlite:///" + sqlite_path							,				cache_dir=lowercase					).read()
   SqlDatasetWriter(lowercase							,				"dataset"							,				"sqlite:///" + output_sqlite_path							,				num_proc=2					).write()

   lowerCAmelCase_:							Optional[Any]       =	iter_sql_file(lowercase					)
   lowerCAmelCase_:							Optional[int]       =	iter_sql_file(lowercase					)

   for rowa, rowa in zip(lowercase							,				lowercase					):
      assert rowa == rowa







@require_sqlalchemy
def 		snake_case__	(		lowercase							,				lowercase							,				lowercase					):
   lowerCAmelCase_:							Union[str, Any]       =	tmp_path / "cache"
   lowerCAmelCase_:							int       =	os.path.join(lowercase							,				"tmp.sql"					)
   lowerCAmelCase_:							Any       =	SqlDatasetReader("dataset"							,				"sqlite:///" + sqlite_path							,				cache_dir=lowercase					).read()
   with pytest.raises(lowercase					):
      SqlDatasetWriter(lowercase							,				"dataset"							,				"sqlite:///" + output_sqlite_path							,				num_proc=0					).write()