# Path: prompt_tuning/data/preprocessors.py
# AUTOTUNE = tf.data.experimental.AUTOTUNE
# def preprocess_tsv_to_qa(line,
#                          field_delim='\t',
#                          answer_delim='|||',
#                          field_names: Optional[Sequence[str]] = None):
# def mrqa(example):
# def preprocess_text_generation(
#     example,
#     source_key,
#     target_key,
#     task_name=None,
#     prefix=None,
#     source_nested_key=None,
#     target_nested_key=None,
# ):
# def remove_first_text_token(example, key: str = 'inputs'):
# def add_sentinel_to_targets(ds, output_features):
# def add_sentinel_to_beginning(ds,
#                               output_features,
#                               field: str = 'targets',
#                               offset: int = 0):
#   def add(ex):
# def build_langid_filter(lang_code, lang_detector, threshold):
#   def filter_fn(text):
# def filter_langid(dataset,
#                   lang_code,
#                   lang_detector,
#                   text_key='text',
#                   threshold=0.95):
# def token_deletion(
#     ds: tf.data.Dataset,
#     sequence_length: SequenceLengthType,
#     output_features: seqio.preprocessors.OutputFeaturesType,
#     noise_density: float = 0.15
# ) -> tf.data.Dataset:
# def text_infilling(
#     ds: tf.data.Dataset,
#     sequence_length: SequenceLengthType,
#     output_features: seqio.preprocessors.OutputFeaturesType,
#     noise_density: float = 0.15,
#     mean_noise_span_length: int = 3
# ) -> tf.data.Dataset:


import os
import textwrap
import unittest.mock as mock
import numpy as np
import seqio
import tensorflow.compat.v2 as tf
import tensorflow_datasets as tfds
from absl.testing import parameterized
from prompt_tuning.data import preprocessors
from seqio import test_utils
           preprocessor=preprocessors.text_infilling),
      dict(testcase_name="token_deletion",
           preprocessor=preprocessors.token_deletion))
  def test_inputs_shorter_than_targets(self, preprocessor):
    ds = create_fake_text_dataset()
    ds = preprocessor(ds,
                      {"inputs": INPUTS_SIZE + 1, "targets": TARGETS_SIZE + 1},
                      TEST_T5_FEATURES,
                      noise_density=0.5)
    for ex in tfds.as_numpy(ds):
      self.assertLess(ex["inputs"].shape[0], ex["targets"].shape[0])

  @parameterized.named_parameters(
      dict(testcase_name="text_infilling",
           preprocessor=preprocessors.text_infilling),
      dict(testcase_name="token_deletion",
           preprocessor=preprocessors.token_deletion))
  def test_extra_id_not_in_targets(self, preprocessor):
    ds = create_fake_text_dataset()
    ds = preprocessor(ds,
                      {"inputs": INPUTS_SIZE + 1, "targets": TARGETS_SIZE + 1},
                      TEST_T5_FEATURES,
                      noise_density=0.5)
    vocab = TEST_T5_FEATURES["targets"].vocabulary
    for ex in tfds.as_numpy(ds):
      targets_text = vocab.decode(ex["targets"].tolist())
      self.assertNotIn("extra_id", targets_text)

  @parameterized.named_parameters(
      dict(testcase_name="text_infilling",

# Path: prompt_tuning/data/preprocessors.py
# AUTOTUNE = tf.data.experimental.AUTOTUNE
# def preprocess_tsv_to_qa(line,
#                          field_delim='\t',
#                          answer_delim='|||',
#                          field_names: Optional[Sequence[str]] = None):
# def mrqa(example):
# def preprocess_text_generation(
#     example,
#     source_key,
#     target_key,
#     task_name=None,
#     prefix=None,
#     source_nested_key=None,
#     target_nested_key=None,
# ):
# def remove_first_text_token(example, key: str = 'inputs'):
# def add_sentinel_to_targets(ds, output_features):
# def add_sentinel_to_beginning(ds,
#                               output_features,
#                               field: str = 'targets',
#                               offset: int = 0):
#   def add(ex):
# def build_langid_filter(lang_code, lang_detector, threshold):
#   def filter_fn(text):
# def filter_langid(dataset,
#                   lang_code,
#                   lang_detector,
#                   text_key='text',
#                   threshold=0.95):
# def token_deletion(
#     ds: tf.data.Dataset,
#     sequence_length: SequenceLengthType,
#     output_features: seqio.preprocessors.OutputFeaturesType,
#     noise_density: float = 0.15
# ) -> tf.data.Dataset:
# def text_infilling(
#     ds: tf.data.Dataset,
#     sequence_length: SequenceLengthType,
#     output_features: seqio.preprocessors.OutputFeaturesType,
#     noise_density: float = 0.15,
#     mean_noise_span_length: int = 3
# ) -> tf.data.Dataset:


# Path: prompt_tuning/data/preprocessors_test.py
import os
import textwrap
import unittest.mock as mock
import numpy as np
import seqio
import tensorflow.compat.v2 as tf
import tensorflow_datasets as tfds
from absl.testing import parameterized
from prompt_tuning.data import preprocessors
from seqio import test_utils

           preprocessor=preprocessors.text_infilling),
      dict(testcase_name="token_deletion",
           preprocessor=preprocessors.token_deletion))
  def test_inputs_shorter_than_targets(self, preprocessor):
    ds = create_fake_text_dataset()
    ds = preprocessor(ds,
                      {"inputs": INPUTS_SIZE + 1, "targets": TARGETS_SIZE + 1},
                      TEST_T5_FEATURES,
                      noise_density=0.5)
    for ex in tfds.as_numpy(ds):
      self.assertLess(ex["inputs"].shape[0], ex["targets"].shape[0])

  @parameterized.named_parameters(
      dict(testcase_name="text_infilling",
           preprocessor=preprocessors.text_infilling),
      dict(testcase_name="token_deletion",
           preprocessor=preprocessors.token_deletion))
  def test_extra_id_not_in_targets(self, preprocessor):
    ds = create_fake_text_dataset()
    ds = preprocessor(ds,
                      {"inputs": INPUTS_SIZE + 1, "targets": TARGETS_SIZE + 1},
                      TEST_T5_FEATURES,
                      noise_density=0.5)
    vocab = TEST_T5_FEATURES["targets"].vocabulary
    for ex in tfds.as_numpy(ds):
      targets_text = vocab.decode(ex["targets"].tolist())
      self.assertNotIn("extra_id", targets_text)

  @parameterized.named_parameters(
      dict(testcase_name="text_infilling",

           preprocessor=preprocessors.text_infilling),