# Path: autoencoder/baseline/doc2vec.py
# class MyDoc2Vec(object):
#     def __init__(self, dim, hs=0, window=5, negative=5, epoches=5, dm=1, dm_concat=1):
#         super(MyDoc2Vec, self).__init__()
#         self.dim = dim
#         self.hs = hs
#         self.window = window
#         self.negative = negative
#         self.epoches = epoches
#         self.dm = dm
#         self.dm_concat = dm_concat
# 
#     def train(self, corpus):
#         self.model = Doc2Vec(min_count=1, window=self.window, size=self.dim, \
#             workers=multiprocessing.cpu_count(), hs=self.hs,\
#             negative=self.negative, iter=1, dm=self.dm, dm_concat=self.dm_concat)
#         self.model.build_vocab(corpus())
#         for each in range(self.epoches):
#             self.model.train(corpus())
# 
#         return self
# 
# def save_doc2vec(model, outfile):
#     model.save(outfile)
# 
# def load_doc2vec(mod_file):
#     return Doc2Vec.load(mod_file)
# 
# def predict(model, corpus):
#     doc_codes = {}
#     for doc_words, doc_name in corpus():
#         doc_codes[doc_name[0]] = model.infer_vector(doc_words).tolist()
# 
#     return doc_codes
# 
# Path: autoencoder/utils/io_utils.py
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e
# 
# def write_file(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             for line in data:
#                 datafile.write(' '.join(line) + '\n')
#     except Exception as e:
#         raise e
# 
# Path: autoencoder/preprocessing/preprocessing.py
# def load_corpus(corpus_path):
#     corpus = load_json(corpus_path)
# 
#     return corpus
# 
# Path: autoencoder/datasets/the20news.py
# class CorpusIter20News(object):
#     def __init__(self, corpus_path, recursive=False, stem=True, with_docname=False):
#         self.stem = stem
#         self.with_docname = with_docname
#         self.files = get_all_files(corpus_path, recursive)
# 
#     def __iter__(self):
#         shuffle(self.files)
#         count = 0
#         for filename in self.files:
#             try:
#                 with open(filename, 'r') as fp:
#                     text = fp.read().lower()
#                     # remove punctuations, stopwords and *unnecessary digits*, stemming
#                     words = tiny_tokenize(text, self.stem, cached_stop_words)
#                     count += 1
#                     if self.with_docname:
#                         parent_name, child_name = os.path.split(filename)
#                         doc_name = os.path.split(parent_name)[-1] + '_' + child_name
#                         yield [words, [doc_name]]
#                     else:
#                         yield words
#             except Exception as e:
#                 raise e
#         print count


import argparse
import timeit
import numpy as np
    import pdb;pdb.set_trace()
    import pdb;pdb.set_trace()
from os import path
from gensim.models.doc2vec import TaggedDocument
from autoencoder.baseline.doc2vec import MyDoc2Vec, save_doc2vec, load_doc2vec, predict
from autoencoder.utils.io_utils import load_json, dump_json, write_file
from autoencoder.preprocessing.preprocessing import load_corpus
from autoencoder.datasets.the20news import CorpusIter20News
from __future__ import absolute_import

# from autoencoder.datasets.movie_review_data import CorpusIterMRD
# from autoencoder.datasets.wiki10plus import CorpusIterWiki10plus
# from autoencoder.datasets.reuters import CorpusIterReuters


def train(args):
    vocab = load_json(args.vocab)
    # import pdb;pdb.set_trace()
    # load corpus
    corpus = CorpusIter20News(args.corpus[0], recursive=True, stem=True, with_docname=True)
    # corpus = CorpusIterMRD(args.corpus[0], load_json(args.docnames), stem=True, with_docname=True)
    # corpus = CorpusIterWiki10plus(args.corpus[0], load_json(args.docnames), stem=True, with_docname=True)
    # corpus = CorpusIterReuters(args.corpus, load_json(args.docnames), with_docname=True)
    corpus_iter = lambda: (TaggedDocument([word for word in sentence if word in vocab], tag) for sentence, tag in corpus)

    d2v = MyDoc2Vec(args.n_dim, window=args.window_size, \
        negative=args.negative, epoches=args.n_epoch, dm_concat=1)

    start = timeit.default_timer()
    d2v.train(corpus_iter)
    print 'runtime: %ss' % (timeit.default_timer() - start)

    save_doc2vec(d2v.model, args.save_model)


def test(args):
    vocab = load_json(args.vocab)
    # load corpus

# Path: autoencoder/baseline/doc2vec.py
# class MyDoc2Vec(object):
#     def __init__(self, dim, hs=0, window=5, negative=5, epoches=5, dm=1, dm_concat=1):
#         super(MyDoc2Vec, self).__init__()
#         self.dim = dim
#         self.hs = hs
#         self.window = window
#         self.negative = negative
#         self.epoches = epoches
#         self.dm = dm
#         self.dm_concat = dm_concat
# 
#     def train(self, corpus):
#         self.model = Doc2Vec(min_count=1, window=self.window, size=self.dim, \
#             workers=multiprocessing.cpu_count(), hs=self.hs,\
#             negative=self.negative, iter=1, dm=self.dm, dm_concat=self.dm_concat)
#         self.model.build_vocab(corpus())
#         for each in range(self.epoches):
#             self.model.train(corpus())
# 
#         return self
# 
# def save_doc2vec(model, outfile):
#     model.save(outfile)
# 
# def load_doc2vec(mod_file):
#     return Doc2Vec.load(mod_file)
# 
# def predict(model, corpus):
#     doc_codes = {}
#     for doc_words, doc_name in corpus():
#         doc_codes[doc_name[0]] = model.infer_vector(doc_words).tolist()
# 
#     return doc_codes
# 
# Path: autoencoder/utils/io_utils.py
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e
# 
# def write_file(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             for line in data:
#                 datafile.write(' '.join(line) + '\n')
#     except Exception as e:
#         raise e
# 
# Path: autoencoder/preprocessing/preprocessing.py
# def load_corpus(corpus_path):
#     corpus = load_json(corpus_path)
# 
#     return corpus
# 
# Path: autoencoder/datasets/the20news.py
# class CorpusIter20News(object):
#     def __init__(self, corpus_path, recursive=False, stem=True, with_docname=False):
#         self.stem = stem
#         self.with_docname = with_docname
#         self.files = get_all_files(corpus_path, recursive)
# 
#     def __iter__(self):
#         shuffle(self.files)
#         count = 0
#         for filename in self.files:
#             try:
#                 with open(filename, 'r') as fp:
#                     text = fp.read().lower()
#                     # remove punctuations, stopwords and *unnecessary digits*, stemming
#                     words = tiny_tokenize(text, self.stem, cached_stop_words)
#                     count += 1
#                     if self.with_docname:
#                         parent_name, child_name = os.path.split(filename)
#                         doc_name = os.path.split(parent_name)[-1] + '_' + child_name
#                         yield [words, [doc_name]]
#                     else:
#                         yield words
#             except Exception as e:
#                 raise e
#         print count


# Path: run_doc2vec.py
import argparse
import timeit
import numpy as np
    import pdb;pdb.set_trace()
    import pdb;pdb.set_trace()
from os import path
from gensim.models.doc2vec import TaggedDocument
from autoencoder.baseline.doc2vec import MyDoc2Vec, save_doc2vec, load_doc2vec, predict
from autoencoder.utils.io_utils import load_json, dump_json, write_file
from autoencoder.preprocessing.preprocessing import load_corpus
from autoencoder.datasets.the20news import CorpusIter20News

from __future__ import absolute_import

# from autoencoder.datasets.movie_review_data import CorpusIterMRD
# from autoencoder.datasets.wiki10plus import CorpusIterWiki10plus
# from autoencoder.datasets.reuters import CorpusIterReuters


def train(args):
    vocab = load_json(args.vocab)
    # import pdb;pdb.set_trace()
    # load corpus
    corpus = CorpusIter20News(args.corpus[0], recursive=True, stem=True, with_docname=True)
    # corpus = CorpusIterMRD(args.corpus[0], load_json(args.docnames), stem=True, with_docname=True)
    # corpus = CorpusIterWiki10plus(args.corpus[0], load_json(args.docnames), stem=True, with_docname=True)
    # corpus = CorpusIterReuters(args.corpus, load_json(args.docnames), with_docname=True)
    corpus_iter = lambda: (TaggedDocument([word for word in sentence if word in vocab], tag) for sentence, tag in corpus)

    d2v = MyDoc2Vec(args.n_dim, window=args.window_size, \
        negative=args.negative, epoches=args.n_epoch, dm_concat=1)

    start = timeit.default_timer()
    d2v.train(corpus_iter)
    print 'runtime: %ss' % (timeit.default_timer() - start)

    save_doc2vec(d2v.model, args.save_model)


def test(args):
    vocab = load_json(args.vocab)
    # load corpus

    corpus = CorpusIter20News(args.corpus[0], recursive=True, stem=True, with_docname=True)