# Path: autoencoder/preprocessing/preprocessing.py
# def build_vocab(word_freq, threshold=5, topn=None, start_idx=0):
#     """
#     threshold only take effects when topn is None.
#     words are indexed by overall frequency in the dataset.
#     """
#     word_freq = sorted(word_freq.iteritems(), key=lambda d:d[1], reverse=True)
#     if topn:
#         word_freq = zip(*word_freq[:topn])[0]
#         vocab_dict = dict(zip(word_freq, range(start_idx, len(word_freq) + start_idx)))
#     else:
#         idx = start_idx
#         vocab_dict = {}
#         for word, freq in word_freq:
#             if freq < threshold:
#                 return vocab_dict
#             vocab_dict[word] = idx
#             idx += 1
#     return vocab_dict
# 
# def generate_bow(doc_word_freq, vocab_dict):
#     docs = {}
#     for key, val in doc_word_freq.iteritems():
#         word_count = {}
#         for word, freq in val.iteritems():
#             try:
#                 word_count[vocab_dict[word]] = freq
#             except: # word is not in vocab, i.e., this word should be discarded
#                 continue
#         docs[key] = word_count
# 
#     return docs
# 
# def count_words(docs):
#     # count the number of times a word appears in a corpus
#     word_freq = defaultdict(lambda: 0)
#     for each in docs:
#         for word, val in each.iteritems():
#             word_freq[word] += val
# 
#     return word_freq
# 
# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


import os
import re
import numpy as np
from random import shuffle
from collections import Counter, defaultdict
from ..preprocessing.preprocessing import build_vocab, generate_bow, count_words
from ..utils.io_utils import dump_json
    np.random.seed(seed)
    np.random.shuffle(corpus)

    n_docs = len(corpus)
    train_data = dict(corpus[:-int(n_docs * test_split)])
    test_data = dict(corpus[-int(n_docs * test_split):])

    return train_data, test_data

def construct_corpus(doc_word_freq, word_freq, training_phase, vocab_dict=None, threshold=5, topn=None):
    if not (training_phase or isinstance(vocab_dict, dict)):
        raise ValueError('vocab_dict must be provided if training_phase is set False')

    if training_phase:
        vocab_dict = build_vocab(word_freq, threshold=threshold, topn=topn)

    docs = generate_bow(doc_word_freq, vocab_dict)
    new_word_freq = dict([(vocab_dict[word], freq) for word, freq in word_freq.iteritems() if word in vocab_dict])

    return docs, vocab_dict, new_word_freq

def construct_train_test_corpus(path_list, test_split, output, threshold=10, topn=20000):
    train_data, test_data = load_data(path_list, test_split)
    train_word_freq = count_words(train_data.values())

    train_docs, vocab_dict, train_word_freq = construct_corpus(train_data, train_word_freq, True, threshold=threshold, topn=topn)
    train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
    dump_json(train_corpus, os.path.join(output, 'train.corpus'))
    print 'Generated training corpus'


# Path: autoencoder/preprocessing/preprocessing.py
# def build_vocab(word_freq, threshold=5, topn=None, start_idx=0):
#     """
#     threshold only take effects when topn is None.
#     words are indexed by overall frequency in the dataset.
#     """
#     word_freq = sorted(word_freq.iteritems(), key=lambda d:d[1], reverse=True)
#     if topn:
#         word_freq = zip(*word_freq[:topn])[0]
#         vocab_dict = dict(zip(word_freq, range(start_idx, len(word_freq) + start_idx)))
#     else:
#         idx = start_idx
#         vocab_dict = {}
#         for word, freq in word_freq:
#             if freq < threshold:
#                 return vocab_dict
#             vocab_dict[word] = idx
#             idx += 1
#     return vocab_dict
# 
# def generate_bow(doc_word_freq, vocab_dict):
#     docs = {}
#     for key, val in doc_word_freq.iteritems():
#         word_count = {}
#         for word, freq in val.iteritems():
#             try:
#                 word_count[vocab_dict[word]] = freq
#             except: # word is not in vocab, i.e., this word should be discarded
#                 continue
#         docs[key] = word_count
# 
#     return docs
# 
# def count_words(docs):
#     # count the number of times a word appears in a corpus
#     word_freq = defaultdict(lambda: 0)
#     for each in docs:
#         for word, val in each.iteritems():
#             word_freq[word] += val
# 
#     return word_freq
# 
# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


# Path: autoencoder/datasets/reuters.py
import os
import re
import numpy as np
from random import shuffle
from collections import Counter, defaultdict
from ..preprocessing.preprocessing import build_vocab, generate_bow, count_words
from ..utils.io_utils import dump_json

    np.random.seed(seed)
    np.random.shuffle(corpus)

    n_docs = len(corpus)
    train_data = dict(corpus[:-int(n_docs * test_split)])
    test_data = dict(corpus[-int(n_docs * test_split):])

    return train_data, test_data

def construct_corpus(doc_word_freq, word_freq, training_phase, vocab_dict=None, threshold=5, topn=None):
    if not (training_phase or isinstance(vocab_dict, dict)):
        raise ValueError('vocab_dict must be provided if training_phase is set False')

    if training_phase:
        vocab_dict = build_vocab(word_freq, threshold=threshold, topn=topn)

    docs = generate_bow(doc_word_freq, vocab_dict)
    new_word_freq = dict([(vocab_dict[word], freq) for word, freq in word_freq.iteritems() if word in vocab_dict])

    return docs, vocab_dict, new_word_freq

def construct_train_test_corpus(path_list, test_split, output, threshold=10, topn=20000):
    train_data, test_data = load_data(path_list, test_split)
    train_word_freq = count_words(train_data.values())

    train_docs, vocab_dict, train_word_freq = construct_corpus(train_data, train_word_freq, True, threshold=threshold, topn=topn)
    train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
    dump_json(train_corpus, os.path.join(output, 'train.corpus'))
    print 'Generated training corpus'


    test_word_freq = count_words(test_data.values())