# Path: autoencoder/preprocessing/preprocessing.py
# def load_corpus(corpus_path):
#     corpus = load_json(corpus_path)
# 
#     return corpus
# 
# def corpus2libsvm(docs, doc_labels, output):
#     '''Convert the corpus format to libsvm format.
#     '''
#     data = []
#     names = []
#     for key, val in docs.iteritems():
#         # label = doc_labels[key]
#         label = 0
#         line = label if isinstance(label, list) else [str(label)] + ["%s:%s" % (int(x) + 1, y) for x, y in val.iteritems()]
#         data.append(line)
#         names.append(key)
#     write_file(data, output)
#     write_file(names, output + '.fnames')
#     return data, names
# 
# Path: autoencoder/utils/io_utils.py
# def load_file(file, float_=False):
#     data = []
#     try:
#         with open(file, 'r') as datafile:
#             for line in datafile:
#                 content = line.strip('\n').split()
#                 if float_:
#                     content = [float(x) for x in content]
#                 data.append(content)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


import os
import sys
import argparse
import numpy as np
    import pdb;pdb.set_trace()
from autoencoder.preprocessing.preprocessing import load_corpus, corpus2libsvm
from autoencoder.utils.io_utils import load_file, dump_json
    args = parser.parse_args()

    train_doc_codes_path = args.train_doc_codes
    test_doc_codes_path = args.test_doc_codes
    train_doc_codes = load_file(train_doc_codes_path, True)
    train_doc_names = load_file(args.train_doc_names)
    val_doc_codes = load_file(args.val_doc_codes, True)
    val_doc_names = load_file(args.val_doc_names)
    test_doc_codes = load_file(test_doc_codes_path, True)
    test_doc_names = load_file(args.test_doc_names)

    assert len(train_doc_codes) == len(train_doc_names)
    assert len(val_doc_codes) == len(val_doc_names)
    assert len(test_doc_codes) == len(test_doc_names)

    new_train_doc_codes = {}
    new_test_doc_codes = {}

    for i in range(len(train_doc_names)):
        new_train_doc_codes[''.join(train_doc_names[i])] = train_doc_codes[i]
    del train_doc_codes
    for i in range(len(val_doc_names)):
        new_train_doc_codes[''.join(val_doc_names[i])] = val_doc_codes[i]
    del val_doc_codes
    for i in range(len(test_doc_names)):
        new_test_doc_codes[''.join(test_doc_names[i])] = test_doc_codes[i]
    del test_doc_codes

    out_dir = args.out_dir
    dump_json(new_train_doc_codes, os.path.join(out_dir, 'new_' + os.path.basename(train_doc_codes_path)))

# Path: autoencoder/preprocessing/preprocessing.py
# def load_corpus(corpus_path):
#     corpus = load_json(corpus_path)
# 
#     return corpus
# 
# def corpus2libsvm(docs, doc_labels, output):
#     '''Convert the corpus format to libsvm format.
#     '''
#     data = []
#     names = []
#     for key, val in docs.iteritems():
#         # label = doc_labels[key]
#         label = 0
#         line = label if isinstance(label, list) else [str(label)] + ["%s:%s" % (int(x) + 1, y) for x, y in val.iteritems()]
#         data.append(line)
#         names.append(key)
#     write_file(data, output)
#     write_file(names, output + '.fnames')
#     return data, names
# 
# Path: autoencoder/utils/io_utils.py
# def load_file(file, float_=False):
#     data = []
#     try:
#         with open(file, 'r') as datafile:
#             for line in datafile:
#                 content = line.strip('\n').split()
#                 if float_:
#                     content = [float(x) for x in content]
#                 data.append(content)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


# Path: docnade_doccodes_converter.py
import os
import sys
import argparse
import numpy as np
    import pdb;pdb.set_trace()
from autoencoder.preprocessing.preprocessing import load_corpus, corpus2libsvm
from autoencoder.utils.io_utils import load_file, dump_json

    args = parser.parse_args()

    train_doc_codes_path = args.train_doc_codes
    test_doc_codes_path = args.test_doc_codes
    train_doc_codes = load_file(train_doc_codes_path, True)
    train_doc_names = load_file(args.train_doc_names)
    val_doc_codes = load_file(args.val_doc_codes, True)
    val_doc_names = load_file(args.val_doc_names)
    test_doc_codes = load_file(test_doc_codes_path, True)
    test_doc_names = load_file(args.test_doc_names)

    assert len(train_doc_codes) == len(train_doc_names)
    assert len(val_doc_codes) == len(val_doc_names)
    assert len(test_doc_codes) == len(test_doc_names)

    new_train_doc_codes = {}
    new_test_doc_codes = {}

    for i in range(len(train_doc_names)):
        new_train_doc_codes[''.join(train_doc_names[i])] = train_doc_codes[i]
    del train_doc_codes
    for i in range(len(val_doc_names)):
        new_train_doc_codes[''.join(val_doc_names[i])] = val_doc_codes[i]
    del val_doc_codes
    for i in range(len(test_doc_names)):
        new_test_doc_codes[''.join(test_doc_names[i])] = test_doc_codes[i]
    del test_doc_codes

    out_dir = args.out_dir
    dump_json(new_train_doc_codes, os.path.join(out_dir, 'new_' + os.path.basename(train_doc_codes_path)))

    dump_json(new_test_doc_codes, os.path.join(out_dir, 'new_' + os.path.basename(test_doc_codes_path)))