# Path: third_party/bi_att_flow/squad/utils.py
# def get_word_span(context, wordss, start, stop):
#   spanss = get_2d_spans(context, wordss)
#   idxs = []
#   for sent_idx, spans in enumerate(spanss):
#     for word_idx, span in enumerate(spans):
#       if not (stop <= span[0] or start >= span[1]):
#         idxs.append((sent_idx, word_idx))
# 
#   assert len(idxs) > 0, "{} {} {} {}".format(context, spanss, start, stop)
#   return idxs[0], (idxs[-1][0], idxs[-1][1] + 1)
# 
# def get_word_idx(context, wordss, idx):
#   spanss = get_2d_spans(context, wordss)
#   return spanss[idx[0]][idx[1]][0]
# 
# def process_tokens(temp_tokens):
#   tokens = []
#   for token in temp_tokens:
#     l = (u"-", u"\u2212", u"\u2014", u"\u2013", u"/", u"~", '"', u"'",
#          u"\u201C", u"\u2019", u"\u201D", u"\u2018", u"\u00B0")
#     # \u2013 is en-dash. Used for number to number
#     # l = ("-", "\u2212", "\u2014", "\u2013")
#     # l = ("\u2013",)
#     tokens.extend(re.split(u"([{}])".format(u"".join(l)), token))
#   return tokens


import argparse
import json
import os
import tensorflow as tf
    import nltk
from collections import Counter
from tqdm import tqdm
from third_party.bi_att_flow.squad.utils import get_word_span, get_word_idx, process_tokens
    from my.corenlp_interface import CoreNLPInterface

          yi.append([yi0, yi1])
          cyi.append([cyi0, cyi1])

        for qij in qi:
          word_counter[qij] += 1
          lower_word_counter[qij.lower()] += 1
          for qijk in qij:
            char_counter[qijk] += 1

        q.append(qi)
        cq.append(cqi)
        y.append(yi)
        cy.append(cyi)
        rx.append(rxi)
        rcx.append(rxi)
        ids.append(qa['id'])
        idxs.append(len(idxs))
        answerss.append(answers)

      if args.debug:
        break

  word2vec_dict = get_word2vec(args, word_counter)
  lower_word2vec_dict = get_word2vec(args, lower_word_counter)

  # add context here
  data = {
      'q': q,
      'cq': cq,

# Path: third_party/bi_att_flow/squad/utils.py
# def get_word_span(context, wordss, start, stop):
#   spanss = get_2d_spans(context, wordss)
#   idxs = []
#   for sent_idx, spans in enumerate(spanss):
#     for word_idx, span in enumerate(spans):
#       if not (stop <= span[0] or start >= span[1]):
#         idxs.append((sent_idx, word_idx))
# 
#   assert len(idxs) > 0, "{} {} {} {}".format(context, spanss, start, stop)
#   return idxs[0], (idxs[-1][0], idxs[-1][1] + 1)
# 
# def get_word_idx(context, wordss, idx):
#   spanss = get_2d_spans(context, wordss)
#   return spanss[idx[0]][idx[1]][0]
# 
# def process_tokens(temp_tokens):
#   tokens = []
#   for token in temp_tokens:
#     l = (u"-", u"\u2212", u"\u2014", u"\u2013", u"/", u"~", '"', u"'",
#          u"\u201C", u"\u2019", u"\u201D", u"\u2018", u"\u00B0")
#     # \u2013 is en-dash. Used for number to number
#     # l = ("-", "\u2212", "\u2014", "\u2013")
#     # l = ("\u2013",)
#     tokens.extend(re.split(u"([{}])".format(u"".join(l)), token))
#   return tokens


# Path: third_party/bi_att_flow/squad/prepro.py
import argparse
import json
import os
import tensorflow as tf
    import nltk
from collections import Counter
from tqdm import tqdm
from third_party.bi_att_flow.squad.utils import get_word_span, get_word_idx, process_tokens
    from my.corenlp_interface import CoreNLPInterface


          yi.append([yi0, yi1])
          cyi.append([cyi0, cyi1])

        for qij in qi:
          word_counter[qij] += 1
          lower_word_counter[qij.lower()] += 1
          for qijk in qij:
            char_counter[qijk] += 1

        q.append(qi)
        cq.append(cqi)
        y.append(yi)
        cy.append(cyi)
        rx.append(rxi)
        rcx.append(rxi)
        ids.append(qa['id'])
        idxs.append(len(idxs))
        answerss.append(answers)

      if args.debug:
        break

  word2vec_dict = get_word2vec(args, word_counter)
  lower_word2vec_dict = get_word2vec(args, lower_word_counter)

  # add context here
  data = {
      'q': q,
      'cq': cq,

      'y': y,