# Path: autoencoder/preprocessing/preprocessing.py
# def load_corpus(corpus_path):
#     corpus = load_json(corpus_path)
# 
#     return corpus
# 
# def doc2vec(doc, dim):
#     vec = np.zeros(dim)
#     for idx, val in doc.items():
#         vec[int(idx)] = val
# 
#     return vec
# 
# Path: autoencoder/utils/io_utils.py
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def dump_pickle(data, path_to_file):
#     try:
#         with open(path_to_file, 'w') as f:
#             pickle.dump(data, f)
#     except Exception as e:
#         raise e


import os
import sys
import argparse
import numpy as np
    import pdb;pdb.set_trace()
from autoencoder.preprocessing.preprocessing import load_corpus, doc2vec
from autoencoder.utils.io_utils import load_json, dump_pickle

    corpus = load_corpus(args.corpus)
    doc_labels = load_json(args.labels)
    vocab, docs = corpus['vocab'], corpus['docs']
    n_vocab = len(vocab)
    doc_names = docs.keys()
    X_docs = [doc2vec(x, n_vocab) for x in docs.values()]

    out_dir = args.out_dir
    # attributes
    attrs = zip(*sorted(vocab.iteritems(), key=lambda d:[1]))[0]
    dump_pickle(attrs, os.path.join(out_dir, 'attributes.p'))

    # batches
    bs = args.batch_size
    batches = [bs * (x + 1) for x in range(int(len(docs) / bs) - 1)]
    batches.append(len(docs))
    dump_pickle(batches, os.path.join(out_dir, 'batches.p'))

    # bow_batch_x
    for i in range(len(batches)):
        dump_pickle(X_docs[batches[i - 1] if i > 0 else 0: batches[i]], os.path.join(out_dir, 'bow_batch_%s.p' % batches[i]))

    # # docs_names_batch_x
    # for i in range(len(batches)):
    #     dump_pickle(doc_names[batches[i - 1] if i > 0 else 0: batches[i]], os.path.join(out_dir, 'docs_names_batch_%s.p' % batches[i]))

    # class_indices_batch_x
    for i in range(len(batches)):
        data = [doc_labels[doc_names[idx]] for idx in range(batches[i - 1] if i > 0 else 0, batches[i])]

# Path: autoencoder/preprocessing/preprocessing.py
# def load_corpus(corpus_path):
#     corpus = load_json(corpus_path)
# 
#     return corpus
# 
# def doc2vec(doc, dim):
#     vec = np.zeros(dim)
#     for idx, val in doc.items():
#         vec[int(idx)] = val
# 
#     return vec
# 
# Path: autoencoder/utils/io_utils.py
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def dump_pickle(data, path_to_file):
#     try:
#         with open(path_to_file, 'w') as f:
#             pickle.dump(data, f)
#     except Exception as e:
#         raise e


# Path: corpus2dbnformat.py
import os
import sys
import argparse
import numpy as np
    import pdb;pdb.set_trace()
from autoencoder.preprocessing.preprocessing import load_corpus, doc2vec
from autoencoder.utils.io_utils import load_json, dump_pickle


    corpus = load_corpus(args.corpus)
    doc_labels = load_json(args.labels)
    vocab, docs = corpus['vocab'], corpus['docs']
    n_vocab = len(vocab)
    doc_names = docs.keys()
    X_docs = [doc2vec(x, n_vocab) for x in docs.values()]

    out_dir = args.out_dir
    # attributes
    attrs = zip(*sorted(vocab.iteritems(), key=lambda d:[1]))[0]
    dump_pickle(attrs, os.path.join(out_dir, 'attributes.p'))

    # batches
    bs = args.batch_size
    batches = [bs * (x + 1) for x in range(int(len(docs) / bs) - 1)]
    batches.append(len(docs))
    dump_pickle(batches, os.path.join(out_dir, 'batches.p'))

    # bow_batch_x
    for i in range(len(batches)):
        dump_pickle(X_docs[batches[i - 1] if i > 0 else 0: batches[i]], os.path.join(out_dir, 'bow_batch_%s.p' % batches[i]))

    # # docs_names_batch_x
    # for i in range(len(batches)):
    #     dump_pickle(doc_names[batches[i - 1] if i > 0 else 0: batches[i]], os.path.join(out_dir, 'docs_names_batch_%s.p' % batches[i]))

    # class_indices_batch_x
    for i in range(len(batches)):
        data = [doc_labels[doc_names[idx]] for idx in range(batches[i - 1] if i > 0 else 0, batches[i])]

        dump_pickle(data, os.path.join(out_dir, 'class_indices_batch_%s.p' % batches[i]))