# Path: data/common.py
# WORD_EMB_PATH = '/path/to/word/embedding/'
# 
# Path: data/bookcorpus.py
# VOCAB_PATH = os.path.join(BOOKCORPUS_DATA_DIR, 'word_count{}')


from gensim.models import KeyedVectors
from data.common import WORD_EMB_PATH
from data.bookcorpus import VOCAB_PATH
import collections
import os
import tqdm
import numpy as np
import sklearn.linear_model
import tensorflow as tf
      vocab[line.split("\t")[0]] = i
      i += 1
  return vocab


def load_pretrained_word_embedding(glove=False):
  print("loading pretrained embedding from disk...")
  word_emb_model = KeyedVectors.load_word2vec_format(
      GLOVE_EMBEDDING_PATH if glove else W2V_EMBEDDING_PATH,
      binary=not glove, limit=1500000)
  return word_emb_model


def expand_vocabulary(model_dir, vocab=None, emb_name="emb_in",
                      use_glove=False, save=False):

  # Load the skip-thoughts embeddings and vocabulary.
  thought_emb = tf.train.load_variable(model_dir, emb_name)

  if vocab is None:
    vocab = load_bookcorpus_vocab()

  # Load the Word2Vec model.
  word2vec = load_pretrained_word_embedding(use_glove)

  # Run vocabulary expansion.
  embedding_map = _expand_vocabulary(thought_emb, vocab, word2vec)

  # Save the output
  words = embedding_map.keys()

# Path: data/common.py
# WORD_EMB_PATH = '/path/to/word/embedding/'
# 
# Path: data/bookcorpus.py
# VOCAB_PATH = os.path.join(BOOKCORPUS_DATA_DIR, 'word_count{}')


# Path: thought/vocabulary_expansion.py
from gensim.models import KeyedVectors
from data.common import WORD_EMB_PATH
from data.bookcorpus import VOCAB_PATH
import collections
import os
import tqdm
import numpy as np
import sklearn.linear_model
import tensorflow as tf

      vocab[line.split("\t")[0]] = i
      i += 1
  return vocab


def load_pretrained_word_embedding(glove=False):
  print("loading pretrained embedding from disk...")
  word_emb_model = KeyedVectors.load_word2vec_format(
      GLOVE_EMBEDDING_PATH if glove else W2V_EMBEDDING_PATH,
      binary=not glove, limit=1500000)
  return word_emb_model


def expand_vocabulary(model_dir, vocab=None, emb_name="emb_in",
                      use_glove=False, save=False):

  # Load the skip-thoughts embeddings and vocabulary.
  thought_emb = tf.train.load_variable(model_dir, emb_name)

  if vocab is None:
    vocab = load_bookcorpus_vocab()

  # Load the Word2Vec model.
  word2vec = load_pretrained_word_embedding(use_glove)

  # Run vocabulary expansion.
  embedding_map = _expand_vocabulary(thought_emb, vocab, word2vec)

  # Save the output
  words = embedding_map.keys()

  embeddings = np.array(embedding_map.values())