# Path: third_party/processors/sb_sed.py
# def infer_sentence_breaks(uni_text):
#   """Generates (start, end) pairs demarking sentences in the text.
# 
#   Args:
#     uni_text: A (multi-sentence) passage of text, in Unicode.
# 
#   Yields:
#     (start, end) tuples that demarcate sentences in the input text. Normal
#     Python slicing applies: the start index points at the first character of
#     the sentence, and the end index is one past the last character of the
#     sentence.
#   """
#   # Treat the text as a single line that starts out with no internal newline
#   # characters and, after regexp-governed substitutions, contains internal
#   # newlines representing cuts between sentences.
#   uni_text = re.sub(r'\n', r' ', uni_text)  # Remove pre-existing newlines.
#   text_with_breaks = _sed_do_sentence_breaks(uni_text)
#   starts = [m.end() for m in re.finditer(r'^\s*', text_with_breaks, re.M)]
#   sentences = [s.strip() for s in text_with_breaks.split('\n')]
#   assert len(starts) == len(sentences)
#   for i in range(len(sentences)):
#     start = starts[i]
#     end = start + len(sentences[i])
#     yield start, end


import json
import logging
import os
import numpy as np
    import torch
    import tensorflow as tf
from functools import partial
from multiprocessing import Pool, cpu_count
from tqdm import tqdm
from transformers.file_utils import is_tf_available, is_torch_available
from transformers.tokenization_bert import whitespace_tokenize
from transformers import DataProcessor
    from torch.utils.data import TensorDataset
from .sb_sed import infer_sentence_breaks
                                                max_answer_length):
    _ = max_seq_length  # TODO Currently we don't use max_seq_length, so make sure that it's being used correctly in tokenization.
    features = []
    # TODO make sure that this tokenization is happening correctly with correct cls, sep and max lengths.
    q_features = tokenizer.encode_plus(
        example.question_text,
        max_length=max_query_length,
        pad_to_max_length=True,
    )
    a_features = tokenizer.encode_plus(
        # Note, this concatenation of sentence and paragraph matches the
        # current preprocessing in evaluate_retrieval.py.
        # TODO(nconstant): Adjust preprocessing here and in
        # evaluate_retrieval.py to pass sentence and paragraph as separate
        # inputs to encode_plus().
        (example.sentence_text + example.paragraph_text).replace("\n", ""),
        max_length=max_answer_length,
        pad_to_max_length=True
    )

    features.append(
        RetrievalSquadFeatures(
            q_features["input_ids"],
            q_features["attention_mask"],
            q_features["token_type_ids"],
            a_features["input_ids"],
            a_features["attention_mask"],
            a_features["token_type_ids"],
            qas_id=example.qas_id,
            example_index=0,

# Path: third_party/processors/sb_sed.py
# def infer_sentence_breaks(uni_text):
#   """Generates (start, end) pairs demarking sentences in the text.
# 
#   Args:
#     uni_text: A (multi-sentence) passage of text, in Unicode.
# 
#   Yields:
#     (start, end) tuples that demarcate sentences in the input text. Normal
#     Python slicing applies: the start index points at the first character of
#     the sentence, and the end index is one past the last character of the
#     sentence.
#   """
#   # Treat the text as a single line that starts out with no internal newline
#   # characters and, after regexp-governed substitutions, contains internal
#   # newlines representing cuts between sentences.
#   uni_text = re.sub(r'\n', r' ', uni_text)  # Remove pre-existing newlines.
#   text_with_breaks = _sed_do_sentence_breaks(uni_text)
#   starts = [m.end() for m in re.finditer(r'^\s*', text_with_breaks, re.M)]
#   sentences = [s.strip() for s in text_with_breaks.split('\n')]
#   assert len(starts) == len(sentences)
#   for i in range(len(sentences)):
#     start = starts[i]
#     end = start + len(sentences[i])
#     yield start, end


# Path: third_party/processors/lareqa.py
import json
import logging
import os
import numpy as np
    import torch
    import tensorflow as tf
from functools import partial
from multiprocessing import Pool, cpu_count
from tqdm import tqdm
from transformers.file_utils import is_tf_available, is_torch_available
from transformers.tokenization_bert import whitespace_tokenize
from transformers import DataProcessor
    from torch.utils.data import TensorDataset
from .sb_sed import infer_sentence_breaks

                                                max_answer_length):
    _ = max_seq_length  # TODO Currently we don't use max_seq_length, so make sure that it's being used correctly in tokenization.
    features = []
    # TODO make sure that this tokenization is happening correctly with correct cls, sep and max lengths.
    q_features = tokenizer.encode_plus(
        example.question_text,
        max_length=max_query_length,
        pad_to_max_length=True,
    )
    a_features = tokenizer.encode_plus(
        # Note, this concatenation of sentence and paragraph matches the
        # current preprocessing in evaluate_retrieval.py.
        # TODO(nconstant): Adjust preprocessing here and in
        # evaluate_retrieval.py to pass sentence and paragraph as separate
        # inputs to encode_plus().
        (example.sentence_text + example.paragraph_text).replace("\n", ""),
        max_length=max_answer_length,
        pad_to_max_length=True
    )

    features.append(
        RetrievalSquadFeatures(
            q_features["input_ids"],
            q_features["attention_mask"],
            q_features["token_type_ids"],
            a_features["input_ids"],
            a_features["attention_mask"],
            a_features["token_type_ids"],
            qas_id=example.qas_id,
            example_index=0,

            unique_id=0,