# Path: third_party/bi_att_flow/squad/utils.py
# def get_word_span(context, wordss, start, stop):
#   spanss = get_2d_spans(context, wordss)
#   idxs = []
#   for sent_idx, spans in enumerate(spanss):
#     for word_idx, span in enumerate(spans):
#       if not (stop <= span[0] or start >= span[1]):
#         idxs.append((sent_idx, word_idx))
# 
#   assert len(idxs) > 0, "{} {} {} {}".format(context, spanss, start, stop)
#   return idxs[0], (idxs[-1][0], idxs[-1][1] + 1)
# 
# def get_word_idx(context, wordss, idx):
#   spanss = get_2d_spans(context, wordss)
#   return spanss[idx[0]][idx[1]][0]
# 
# def process_tokens(temp_tokens):
#   tokens = []
#   for token in temp_tokens:
#     l = (u"-", u"\u2212", u"\u2014", u"\u2013", u"/", u"~", '"', u"'",
#          u"\u201C", u"\u2019", u"\u201D", u"\u2018", u"\u00B0")
#     # \u2013 is en-dash. Used for number to number
#     # l = ("-", "\u2212", "\u2014", "\u2013")
#     # l = ("\u2013",)
#     tokens.extend(re.split(u"([{}])".format(u"".join(l)), token))
#   return tokens


import argparse
import json
import os
import tensorflow as tf
    import nltk
from collections import Counter
from tqdm import tqdm
from third_party.bi_att_flow.squad.utils import get_word_span, get_word_idx, process_tokens
    from my.corenlp_interface import CoreNLPInterface
def save(args, data, shared, data_type):
  data_path = os.path.join(args.target_dir, 'data_{}.json'.format(data_type))
  shared_path = os.path.join(args.target_dir,
                             'shared_{}.json'.format(data_type))
  json.dump(data, tf.gfile.Open(data_path, 'wb'))
  json.dump(shared, tf.gfile.Open(shared_path, 'wb'))


def get_word2vec(args, word_counter):
  glove_path = os.path.join(args.glove_dir, 'glove.{}.{}d.txt'.format(
      args.glove_corpus, args.glove_vec_size))
  sizes = {
      '6B': int(4e5),
      '42B': int(1.9e6),
      '840B': int(2.2e6),
      '2B': int(1.2e6)
  }
  total = sizes[args.glove_corpus]
  word2vec_dict = {}
  with tf.gfile.Open(glove_path, 'r') as fh:
    for line in tqdm(fh, total=total):
      array = line.lstrip().rstrip().split(' ')
      word = array[0]
      vector = list(map(float, array[1:]))
      if word in word_counter:
        word2vec_dict[word] = vector
      elif word.capitalize() in word_counter:
        word2vec_dict[word.capitalize()] = vector
      elif word.lower() in word_counter:
        word2vec_dict[word.lower()] = vector

# Path: third_party/bi_att_flow/squad/utils.py
# def get_word_span(context, wordss, start, stop):
#   spanss = get_2d_spans(context, wordss)
#   idxs = []
#   for sent_idx, spans in enumerate(spanss):
#     for word_idx, span in enumerate(spans):
#       if not (stop <= span[0] or start >= span[1]):
#         idxs.append((sent_idx, word_idx))
# 
#   assert len(idxs) > 0, "{} {} {} {}".format(context, spanss, start, stop)
#   return idxs[0], (idxs[-1][0], idxs[-1][1] + 1)
# 
# def get_word_idx(context, wordss, idx):
#   spanss = get_2d_spans(context, wordss)
#   return spanss[idx[0]][idx[1]][0]
# 
# def process_tokens(temp_tokens):
#   tokens = []
#   for token in temp_tokens:
#     l = (u"-", u"\u2212", u"\u2014", u"\u2013", u"/", u"~", '"', u"'",
#          u"\u201C", u"\u2019", u"\u201D", u"\u2018", u"\u00B0")
#     # \u2013 is en-dash. Used for number to number
#     # l = ("-", "\u2212", "\u2014", "\u2013")
#     # l = ("\u2013",)
#     tokens.extend(re.split(u"([{}])".format(u"".join(l)), token))
#   return tokens


# Path: third_party/bi_att_flow/squad/prepro.py
import argparse
import json
import os
import tensorflow as tf
    import nltk
from collections import Counter
from tqdm import tqdm
from third_party.bi_att_flow.squad.utils import get_word_span, get_word_idx, process_tokens
    from my.corenlp_interface import CoreNLPInterface

def save(args, data, shared, data_type):
  data_path = os.path.join(args.target_dir, 'data_{}.json'.format(data_type))
  shared_path = os.path.join(args.target_dir,
                             'shared_{}.json'.format(data_type))
  json.dump(data, tf.gfile.Open(data_path, 'wb'))
  json.dump(shared, tf.gfile.Open(shared_path, 'wb'))


def get_word2vec(args, word_counter):
  glove_path = os.path.join(args.glove_dir, 'glove.{}.{}d.txt'.format(
      args.glove_corpus, args.glove_vec_size))
  sizes = {
      '6B': int(4e5),
      '42B': int(1.9e6),
      '840B': int(2.2e6),
      '2B': int(1.2e6)
  }
  total = sizes[args.glove_corpus]
  word2vec_dict = {}
  with tf.gfile.Open(glove_path, 'r') as fh:
    for line in tqdm(fh, total=total):
      array = line.lstrip().rstrip().split(' ')
      word = array[0]
      vector = list(map(float, array[1:]))
      if word in word_counter:
        word2vec_dict[word] = vector
      elif word.capitalize() in word_counter:
        word2vec_dict[word.capitalize()] = vector
      elif word.lower() in word_counter:
        word2vec_dict[word.lower()] = vector

      elif word.upper() in word_counter: