# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e
# 
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def write_file(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             for line in data:
#                 datafile.write(' '.join(line) + '\n')
#     except Exception as e:
#         raise e


import os
import re
import string
import codecs
import numpy as np
from collections import defaultdict
from nltk.tokenize import wordpunct_tokenize
from nltk.stem.porter import PorterStemmer as EnglishStemmer
from ..utils.io_utils import dump_json, load_json, write_file
        from nltk.corpus import stopwords

    return docs

def build_vocab(word_freq, threshold=5, topn=None, start_idx=0):
    """
    threshold only take effects when topn is None.
    words are indexed by overall frequency in the dataset.
    """
    word_freq = sorted(word_freq.iteritems(), key=lambda d:d[1], reverse=True)
    if topn:
        word_freq = zip(*word_freq[:topn])[0]
        vocab_dict = dict(zip(word_freq, range(start_idx, len(word_freq) + start_idx)))
    else:
        idx = start_idx
        vocab_dict = {}
        for word, freq in word_freq:
            if freq < threshold:
                return vocab_dict
            vocab_dict[word] = idx
            idx += 1
    return vocab_dict

def construct_train_test_corpus(train_path, test_path, output, threshold=5, topn=None):
    train_docs, vocab_dict, train_word_freq = construct_corpus(train_path, True, threshold=threshold, topn=topn, recursive=True)
    train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
    dump_json(train_corpus, os.path.join(output, 'train.corpus'))
    print 'Generated training corpus'

    test_docs, _, _ = construct_corpus(test_path, False, vocab_dict=vocab_dict, recursive=True)
    test_corpus = {'docs': test_docs, 'vocab': vocab_dict}

# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e
# 
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def write_file(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             for line in data:
#                 datafile.write(' '.join(line) + '\n')
#     except Exception as e:
#         raise e


# Path: autoencoder/preprocessing/preprocessing.py
import os
import re
import string
import codecs
import numpy as np
from collections import defaultdict
from nltk.tokenize import wordpunct_tokenize
from nltk.stem.porter import PorterStemmer as EnglishStemmer
from ..utils.io_utils import dump_json, load_json, write_file
        from nltk.corpus import stopwords


    return docs

def build_vocab(word_freq, threshold=5, topn=None, start_idx=0):
    """
    threshold only take effects when topn is None.
    words are indexed by overall frequency in the dataset.
    """
    word_freq = sorted(word_freq.iteritems(), key=lambda d:d[1], reverse=True)
    if topn:
        word_freq = zip(*word_freq[:topn])[0]
        vocab_dict = dict(zip(word_freq, range(start_idx, len(word_freq) + start_idx)))
    else:
        idx = start_idx
        vocab_dict = {}
        for word, freq in word_freq:
            if freq < threshold:
                return vocab_dict
            vocab_dict[word] = idx
            idx += 1
    return vocab_dict

def construct_train_test_corpus(train_path, test_path, output, threshold=5, topn=None):
    train_docs, vocab_dict, train_word_freq = construct_corpus(train_path, True, threshold=threshold, topn=topn, recursive=True)
    train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
    dump_json(train_corpus, os.path.join(output, 'train.corpus'))
    print 'Generated training corpus'

    test_docs, _, _ = construct_corpus(test_path, False, vocab_dict=vocab_dict, recursive=True)
    test_corpus = {'docs': test_docs, 'vocab': vocab_dict}

    dump_json(test_corpus, os.path.join(output, 'test.corpus'))