# Path: third_party/bi_att_flow/squad/utils.py
# def get_word_span(context, wordss, start, stop):
#   spanss = get_2d_spans(context, wordss)
#   idxs = []
#   for sent_idx, spans in enumerate(spanss):
#     for word_idx, span in enumerate(spans):
#       if not (stop <= span[0] or start >= span[1]):
#         idxs.append((sent_idx, word_idx))
# 
#   assert len(idxs) > 0, "{} {} {} {}".format(context, spanss, start, stop)
#   return idxs[0], (idxs[-1][0], idxs[-1][1] + 1)
# 
# def get_word_idx(context, wordss, idx):
#   spanss = get_2d_spans(context, wordss)
#   return spanss[idx[0]][idx[1]][0]
# 
# def process_tokens(temp_tokens):
#   tokens = []
#   for token in temp_tokens:
#     l = (u"-", u"\u2212", u"\u2014", u"\u2013", u"/", u"~", '"', u"'",
#          u"\u201C", u"\u2019", u"\u201D", u"\u2018", u"\u00B0")
#     # \u2013 is en-dash. Used for number to number
#     # l = ("-", "\u2212", "\u2014", "\u2013")
#     # l = ("\u2013",)
#     tokens.extend(re.split(u"([{}])".format(u"".join(l)), token))
#   return tokens


import argparse
import json
import os
import tensorflow as tf
    import nltk
from collections import Counter
from tqdm import tqdm
from third_party.bi_att_flow.squad.utils import get_word_span, get_word_idx, process_tokens
    from my.corenlp_interface import CoreNLPInterface
      for xij in xi:
        for xijk in xij:
          word_counter[xijk] += len(para['qas'])
          lower_word_counter[xijk.lower()] += len(para['qas'])
          for xijkl in xijk:
            char_counter[xijkl] += len(para['qas'])

      rxi = [ai, pi]
      assert len(x) - 1 == ai
      assert len(x[ai]) - 1 == pi
      for qa in para['qas']:
        # get words
        qi = word_tokenize(qa['question'])
        cqi = [list(qij) for qij in qi]
        yi = []
        cyi = []
        answers = []
        for answer in qa['answers']:
          answer_text = answer['text']
          answers.append(answer_text)
          answer_start = answer['answer_start']
          answer_stop = answer_start + len(answer_text)
          yi0, yi1 = get_word_span(context, xi, answer_start, answer_stop)
          # yi0 = answer['answer_word_start'] or [0, 0]
          # yi1 = answer['answer_word_stop'] or [0, 1]
          assert len(xi[yi0[0]]) > yi0[1]
          assert len(xi[yi1[0]]) >= yi1[1]
          w0 = xi[yi0[0]][yi0[1]]
          w1 = xi[yi1[0]][yi1[1] - 1]
          i0 = get_word_idx(context, xi, yi0)

# Path: third_party/bi_att_flow/squad/utils.py
# def get_word_span(context, wordss, start, stop):
#   spanss = get_2d_spans(context, wordss)
#   idxs = []
#   for sent_idx, spans in enumerate(spanss):
#     for word_idx, span in enumerate(spans):
#       if not (stop <= span[0] or start >= span[1]):
#         idxs.append((sent_idx, word_idx))
# 
#   assert len(idxs) > 0, "{} {} {} {}".format(context, spanss, start, stop)
#   return idxs[0], (idxs[-1][0], idxs[-1][1] + 1)
# 
# def get_word_idx(context, wordss, idx):
#   spanss = get_2d_spans(context, wordss)
#   return spanss[idx[0]][idx[1]][0]
# 
# def process_tokens(temp_tokens):
#   tokens = []
#   for token in temp_tokens:
#     l = (u"-", u"\u2212", u"\u2014", u"\u2013", u"/", u"~", '"', u"'",
#          u"\u201C", u"\u2019", u"\u201D", u"\u2018", u"\u00B0")
#     # \u2013 is en-dash. Used for number to number
#     # l = ("-", "\u2212", "\u2014", "\u2013")
#     # l = ("\u2013",)
#     tokens.extend(re.split(u"([{}])".format(u"".join(l)), token))
#   return tokens


# Path: third_party/bi_att_flow/squad/prepro.py
import argparse
import json
import os
import tensorflow as tf
    import nltk
from collections import Counter
from tqdm import tqdm
from third_party.bi_att_flow.squad.utils import get_word_span, get_word_idx, process_tokens
    from my.corenlp_interface import CoreNLPInterface

      for xij in xi:
        for xijk in xij:
          word_counter[xijk] += len(para['qas'])
          lower_word_counter[xijk.lower()] += len(para['qas'])
          for xijkl in xijk:
            char_counter[xijkl] += len(para['qas'])

      rxi = [ai, pi]
      assert len(x) - 1 == ai
      assert len(x[ai]) - 1 == pi
      for qa in para['qas']:
        # get words
        qi = word_tokenize(qa['question'])
        cqi = [list(qij) for qij in qi]
        yi = []
        cyi = []
        answers = []
        for answer in qa['answers']:
          answer_text = answer['text']
          answers.append(answer_text)
          answer_start = answer['answer_start']
          answer_stop = answer_start + len(answer_text)
          yi0, yi1 = get_word_span(context, xi, answer_start, answer_stop)
          # yi0 = answer['answer_word_start'] or [0, 0]
          # yi1 = answer['answer_word_stop'] or [0, 1]
          assert len(xi[yi0[0]]) > yi0[1]
          assert len(xi[yi1[0]]) >= yi1[1]
          w0 = xi[yi0[0]][yi0[1]]
          w1 = xi[yi1[0]][yi1[1] - 1]
          i0 = get_word_idx(context, xi, yi0)

          i1 = get_word_idx(context, xi, (yi1[0], yi1[1] - 1))