# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e
# 
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def write_file(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             for line in data:
#                 datafile.write(' '.join(line) + '\n')
#     except Exception as e:
#         raise e


import os
import re
import string
import codecs
import numpy as np
from collections import defaultdict
from nltk.tokenize import wordpunct_tokenize
from nltk.stem.porter import PorterStemmer as EnglishStemmer
from ..utils.io_utils import dump_json, load_json, write_file
        from nltk.corpus import stopwords
                # doc_name = os.path.basename(filename)
                parent_name, child_name = os.path.split(filename)
                doc_name = os.path.split(parent_name)[-1] + '_' + child_name
                for i in range(len(words)):
                    # doc-word frequency
                    try:
                        doc_word_freq[doc_name][words[i]] += 1
                    except:
                        doc_word_freq[doc_name][words[i]] = 1
                    # word frequency
                    word_freq[words[i]] += 1
        except Exception as e:
            raise e

    return word_freq, doc_word_freq

def construct_corpus(corpus_path, training_phase, vocab_dict=None, threshold=5, topn=None, recursive=False):
    if not (training_phase or isinstance(vocab_dict, dict)):
        raise ValueError('vocab_dict must be provided if training_phase is set False')

    word_freq, doc_word_freq = load_data(corpus_path, recursive)
    if training_phase:
        vocab_dict = build_vocab(word_freq, threshold=threshold, topn=topn)

    docs = generate_bow(doc_word_freq, vocab_dict)
    new_word_freq = dict([(vocab_dict[word], freq) for word, freq in word_freq.iteritems() if word in vocab_dict])

    return docs, vocab_dict, new_word_freq

def load_corpus(corpus_path):

# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e
# 
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data
# 
# def write_file(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             for line in data:
#                 datafile.write(' '.join(line) + '\n')
#     except Exception as e:
#         raise e


# Path: autoencoder/preprocessing/preprocessing.py
import os
import re
import string
import codecs
import numpy as np
from collections import defaultdict
from nltk.tokenize import wordpunct_tokenize
from nltk.stem.porter import PorterStemmer as EnglishStemmer
from ..utils.io_utils import dump_json, load_json, write_file
        from nltk.corpus import stopwords

                # doc_name = os.path.basename(filename)
                parent_name, child_name = os.path.split(filename)
                doc_name = os.path.split(parent_name)[-1] + '_' + child_name
                for i in range(len(words)):
                    # doc-word frequency
                    try:
                        doc_word_freq[doc_name][words[i]] += 1
                    except:
                        doc_word_freq[doc_name][words[i]] = 1
                    # word frequency
                    word_freq[words[i]] += 1
        except Exception as e:
            raise e

    return word_freq, doc_word_freq

def construct_corpus(corpus_path, training_phase, vocab_dict=None, threshold=5, topn=None, recursive=False):
    if not (training_phase or isinstance(vocab_dict, dict)):
        raise ValueError('vocab_dict must be provided if training_phase is set False')

    word_freq, doc_word_freq = load_data(corpus_path, recursive)
    if training_phase:
        vocab_dict = build_vocab(word_freq, threshold=threshold, topn=topn)

    docs = generate_bow(doc_word_freq, vocab_dict)
    new_word_freq = dict([(vocab_dict[word], freq) for word, freq in word_freq.iteritems() if word in vocab_dict])

    return docs, vocab_dict, new_word_freq

def load_corpus(corpus_path):

    corpus = load_json(corpus_path)