# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e
# 
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def write_file(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             for line in data:
#                 datafile.write(' '.join(line) + '\n')
#     except Exception as e:
#         raise e


import os
import re
import string
import codecs
import numpy as np
from collections import defaultdict
from nltk.tokenize import wordpunct_tokenize
from nltk.stem.porter import PorterStemmer as EnglishStemmer
from ..utils.io_utils import dump_json, load_json, write_file
        from nltk.corpus import stopwords
                return vocab_dict
            vocab_dict[word] = idx
            idx += 1
    return vocab_dict

def construct_train_test_corpus(train_path, test_path, output, threshold=5, topn=None):
    train_docs, vocab_dict, train_word_freq = construct_corpus(train_path, True, threshold=threshold, topn=topn, recursive=True)
    train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
    dump_json(train_corpus, os.path.join(output, 'train.corpus'))
    print 'Generated training corpus'

    test_docs, _, _ = construct_corpus(test_path, False, vocab_dict=vocab_dict, recursive=True)
    test_corpus = {'docs': test_docs, 'vocab': vocab_dict}
    dump_json(test_corpus, os.path.join(output, 'test.corpus'))
    print 'Generated test corpus'

    return train_corpus, test_corpus

def corpus2libsvm(docs, doc_labels, output):
    '''Convert the corpus format to libsvm format.
    '''
    data = []
    names = []
    for key, val in docs.iteritems():
        # label = doc_labels[key]
        label = 0
        line = label if isinstance(label, list) else [str(label)] + ["%s:%s" % (int(x) + 1, y) for x, y in val.iteritems()]
        data.append(line)
        names.append(key)
    write_file(data, output)

# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e
# 
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def write_file(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             for line in data:
#                 datafile.write(' '.join(line) + '\n')
#     except Exception as e:
#         raise e


# Path: autoencoder/preprocessing/preprocessing.py
import os
import re
import string
import codecs
import numpy as np
from collections import defaultdict
from nltk.tokenize import wordpunct_tokenize
from nltk.stem.porter import PorterStemmer as EnglishStemmer
from ..utils.io_utils import dump_json, load_json, write_file
        from nltk.corpus import stopwords

                return vocab_dict
            vocab_dict[word] = idx
            idx += 1
    return vocab_dict

def construct_train_test_corpus(train_path, test_path, output, threshold=5, topn=None):
    train_docs, vocab_dict, train_word_freq = construct_corpus(train_path, True, threshold=threshold, topn=topn, recursive=True)
    train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
    dump_json(train_corpus, os.path.join(output, 'train.corpus'))
    print 'Generated training corpus'

    test_docs, _, _ = construct_corpus(test_path, False, vocab_dict=vocab_dict, recursive=True)
    test_corpus = {'docs': test_docs, 'vocab': vocab_dict}
    dump_json(test_corpus, os.path.join(output, 'test.corpus'))
    print 'Generated test corpus'

    return train_corpus, test_corpus

def corpus2libsvm(docs, doc_labels, output):
    '''Convert the corpus format to libsvm format.
    '''
    data = []
    names = []
    for key, val in docs.iteritems():
        # label = doc_labels[key]
        label = 0
        line = label if isinstance(label, list) else [str(label)] + ["%s:%s" % (int(x) + 1, y) for x, y in val.iteritems()]
        data.append(line)
        names.append(key)
    write_file(data, output)

    write_file(names, output + '.fnames')