# Path: third_party/bi_att_flow/squad/utils.py
# def get_word_span(context, wordss, start, stop):
#   spanss = get_2d_spans(context, wordss)
#   idxs = []
#   for sent_idx, spans in enumerate(spanss):
#     for word_idx, span in enumerate(spans):
#       if not (stop <= span[0] or start >= span[1]):
#         idxs.append((sent_idx, word_idx))
# 
#   assert len(idxs) > 0, "{} {} {} {}".format(context, spanss, start, stop)
#   return idxs[0], (idxs[-1][0], idxs[-1][1] + 1)
# 
# def get_word_idx(context, wordss, idx):
#   spanss = get_2d_spans(context, wordss)
#   return spanss[idx[0]][idx[1]][0]
# 
# def process_tokens(temp_tokens):
#   tokens = []
#   for token in temp_tokens:
#     l = (u"-", u"\u2212", u"\u2014", u"\u2013", u"/", u"~", '"', u"'",
#          u"\u201C", u"\u2019", u"\u201D", u"\u2018", u"\u00B0")
#     # \u2013 is en-dash. Used for number to number
#     # l = ("-", "\u2212", "\u2014", "\u2013")
#     # l = ("\u2013",)
#     tokens.extend(re.split(u"([{}])".format(u"".join(l)), token))
#   return tokens


import argparse
import json
import os
import tensorflow as tf
    import nltk
from collections import Counter
from tqdm import tqdm
from third_party.bi_att_flow.squad.utils import get_word_span, get_word_idx, process_tokens
    from my.corenlp_interface import CoreNLPInterface
    raise Exception()

  if not args.split:
    sent_tokenize = lambda para: [para]

  source_path = in_path or os.path.join(args.source_dir,
                                        '{}-v1.1.json'.format(data_type))
  source_data = json.load(tf.gfile.Open(source_path, 'r'))

  q, cq, y, rx, rcx, ids, idxs = [], [], [], [], [], [], []
  cy = []
  x, cx = [], []
  answerss = []
  p = []
  word_counter, char_counter, lower_word_counter = Counter(), Counter(
  ), Counter()
  start_ai = int(round(len(source_data['data']) * start_ratio))
  stop_ai = int(round(len(source_data['data']) * stop_ratio))
  for ai, article in enumerate(tqdm(source_data['data'][start_ai:stop_ai])):
    xp, cxp = [], []
    pp = []
    x.append(xp)
    cx.append(cxp)
    p.append(pp)
    for pi, para in enumerate(article['paragraphs']):
      # wordss
      context = para['context']
      context = context.replace("''", '" ')
      context = context.replace('``', '" ')
      xi = list(map(word_tokenize, sent_tokenize(context)))

# Path: third_party/bi_att_flow/squad/utils.py
# def get_word_span(context, wordss, start, stop):
#   spanss = get_2d_spans(context, wordss)
#   idxs = []
#   for sent_idx, spans in enumerate(spanss):
#     for word_idx, span in enumerate(spans):
#       if not (stop <= span[0] or start >= span[1]):
#         idxs.append((sent_idx, word_idx))
# 
#   assert len(idxs) > 0, "{} {} {} {}".format(context, spanss, start, stop)
#   return idxs[0], (idxs[-1][0], idxs[-1][1] + 1)
# 
# def get_word_idx(context, wordss, idx):
#   spanss = get_2d_spans(context, wordss)
#   return spanss[idx[0]][idx[1]][0]
# 
# def process_tokens(temp_tokens):
#   tokens = []
#   for token in temp_tokens:
#     l = (u"-", u"\u2212", u"\u2014", u"\u2013", u"/", u"~", '"', u"'",
#          u"\u201C", u"\u2019", u"\u201D", u"\u2018", u"\u00B0")
#     # \u2013 is en-dash. Used for number to number
#     # l = ("-", "\u2212", "\u2014", "\u2013")
#     # l = ("\u2013",)
#     tokens.extend(re.split(u"([{}])".format(u"".join(l)), token))
#   return tokens


# Path: third_party/bi_att_flow/squad/prepro.py
import argparse
import json
import os
import tensorflow as tf
    import nltk
from collections import Counter
from tqdm import tqdm
from third_party.bi_att_flow.squad.utils import get_word_span, get_word_idx, process_tokens
    from my.corenlp_interface import CoreNLPInterface

    raise Exception()

  if not args.split:
    sent_tokenize = lambda para: [para]

  source_path = in_path or os.path.join(args.source_dir,
                                        '{}-v1.1.json'.format(data_type))
  source_data = json.load(tf.gfile.Open(source_path, 'r'))

  q, cq, y, rx, rcx, ids, idxs = [], [], [], [], [], [], []
  cy = []
  x, cx = [], []
  answerss = []
  p = []
  word_counter, char_counter, lower_word_counter = Counter(), Counter(
  ), Counter()
  start_ai = int(round(len(source_data['data']) * start_ratio))
  stop_ai = int(round(len(source_data['data']) * stop_ratio))
  for ai, article in enumerate(tqdm(source_data['data'][start_ai:stop_ai])):
    xp, cxp = [], []
    pp = []
    x.append(xp)
    cx.append(cxp)
    p.append(pp)
    for pi, para in enumerate(article['paragraphs']):
      # wordss
      context = para['context']
      context = context.replace("''", '" ')
      context = context.replace('``', '" ')
      xi = list(map(word_tokenize, sent_tokenize(context)))

      xi = [process_tokens(tokens) for tokens in xi]  # process tokens