# Path: prompt_tuning/data/features.py
# T5_FEATURES = t5_tasks.DEFAULT_OUTPUT_FEATURES
# MODEL_TO_FEATURES = {
#     # We use the keys to prefix the tasks t5 is the default so we don't want a
#     # prefix, thus the strange empty string as the key.
#     "": T5_FEATURES,
# }
#   MT5_FEATURES = mt5_tasks.DEFAULT_OUTPUT_FEATURES
#   BYT5_FEATURES = byt5_tasks.DEFAULT_BYTE_OUTPUT_FEATURES
# 
# Path: prompt_tuning/data/preprocessors.py
# AUTOTUNE = tf.data.experimental.AUTOTUNE
# def preprocess_tsv_to_qa(line,
#                          field_delim='\t',
#                          answer_delim='|||',
#                          field_names: Optional[Sequence[str]] = None):
# def mrqa(example):
# def preprocess_text_generation(
#     example,
#     source_key,
#     target_key,
#     task_name=None,
#     prefix=None,
#     source_nested_key=None,
#     target_nested_key=None,
# ):
# def remove_first_text_token(example, key: str = 'inputs'):
# def add_sentinel_to_targets(ds, output_features):
# def add_sentinel_to_beginning(ds,
#                               output_features,
#                               field: str = 'targets',
#                               offset: int = 0):
#   def add(ex):
# def build_langid_filter(lang_code, lang_detector, threshold):
#   def filter_fn(text):
# def filter_langid(dataset,
#                   lang_code,
#                   lang_detector,
#                   text_key='text',
#                   threshold=0.95):
# def token_deletion(
#     ds: tf.data.Dataset,
#     sequence_length: SequenceLengthType,
#     output_features: seqio.preprocessors.OutputFeaturesType,
#     noise_density: float = 0.15
# ) -> tf.data.Dataset:
# def text_infilling(
#     ds: tf.data.Dataset,
#     sequence_length: SequenceLengthType,
#     output_features: seqio.preprocessors.OutputFeaturesType,
#     noise_density: float = 0.15,
#     mean_noise_span_length: int = 3
# ) -> tf.data.Dataset:


import functools
import seqio
from prompt_tuning.data import features
from prompt_tuning.data import preprocessors
from t5.data import preprocessors as t5_preprocessors
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tasks that use the c4 dataset."""




# ===== BART (Lweis et. al., 2019) Pre-training-like objectives =====
seqio.TaskRegistry.add(
    "c4_v220_bart_text_infilling",
    source=seqio.TfdsDataSource(tfds_name="c4/en:2.2.0"),
    preprocessors=[
        functools.partial(
            t5_preprocessors.rekey,
            key_map={
                "inputs": None,
                "targets": "text",
            }),
        seqio.preprocessors.tokenize,
        seqio.CacheDatasetPlaceholder(),
        preprocessors.text_infilling,

# Path: prompt_tuning/data/features.py
# T5_FEATURES = t5_tasks.DEFAULT_OUTPUT_FEATURES
# MODEL_TO_FEATURES = {
#     # We use the keys to prefix the tasks t5 is the default so we don't want a
#     # prefix, thus the strange empty string as the key.
#     "": T5_FEATURES,
# }
#   MT5_FEATURES = mt5_tasks.DEFAULT_OUTPUT_FEATURES
#   BYT5_FEATURES = byt5_tasks.DEFAULT_BYTE_OUTPUT_FEATURES
# 
# Path: prompt_tuning/data/preprocessors.py
# AUTOTUNE = tf.data.experimental.AUTOTUNE
# def preprocess_tsv_to_qa(line,
#                          field_delim='\t',
#                          answer_delim='|||',
#                          field_names: Optional[Sequence[str]] = None):
# def mrqa(example):
# def preprocess_text_generation(
#     example,
#     source_key,
#     target_key,
#     task_name=None,
#     prefix=None,
#     source_nested_key=None,
#     target_nested_key=None,
# ):
# def remove_first_text_token(example, key: str = 'inputs'):
# def add_sentinel_to_targets(ds, output_features):
# def add_sentinel_to_beginning(ds,
#                               output_features,
#                               field: str = 'targets',
#                               offset: int = 0):
#   def add(ex):
# def build_langid_filter(lang_code, lang_detector, threshold):
#   def filter_fn(text):
# def filter_langid(dataset,
#                   lang_code,
#                   lang_detector,
#                   text_key='text',
#                   threshold=0.95):
# def token_deletion(
#     ds: tf.data.Dataset,
#     sequence_length: SequenceLengthType,
#     output_features: seqio.preprocessors.OutputFeaturesType,
#     noise_density: float = 0.15
# ) -> tf.data.Dataset:
# def text_infilling(
#     ds: tf.data.Dataset,
#     sequence_length: SequenceLengthType,
#     output_features: seqio.preprocessors.OutputFeaturesType,
#     noise_density: float = 0.15,
#     mean_noise_span_length: int = 3
# ) -> tf.data.Dataset:


# Path: prompt_tuning/data/c4.py
import functools
import seqio
from prompt_tuning.data import features
from prompt_tuning.data import preprocessors
from t5.data import preprocessors as t5_preprocessors

# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tasks that use the c4 dataset."""




# ===== BART (Lweis et. al., 2019) Pre-training-like objectives =====
seqio.TaskRegistry.add(
    "c4_v220_bart_text_infilling",
    source=seqio.TfdsDataSource(tfds_name="c4/en:2.2.0"),
    preprocessors=[
        functools.partial(
            t5_preprocessors.rekey,
            key_map={
                "inputs": None,
                "targets": "text",
            }),
        seqio.preprocessors.tokenize,
        seqio.CacheDatasetPlaceholder(),
        preprocessors.text_infilling,

        seqio.preprocessors.append_eos_after_trim,