# Path: autoencoder/preprocessing/preprocessing.py
# def init_stopwords():
#     try:
#         stopword_path = 'patterns/english_stopwords.txt'
#         cached_stop_words = load_stopwords(os.path.join(os.path.split(__file__)[0], stopword_path))
#         print 'Loaded %s' % stopword_path
#     except:
#         from nltk.corpus import stopwords
#         cached_stop_words = stopwords.words("english")
#         print 'Loaded nltk.corpus.stopwords'
# 
#     return cached_stop_words
# 
# def tiny_tokenize_xml(text, stem=False, stop_words=[]):
#     return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
#                         re.sub('[%s]' % re.escape(string.punctuation), ' ', text.encode(encoding='ascii', errors='ignore'))) if
#                         not token.isdigit() and not token in stop_words]
# 
# def tiny_tokenize(text, stem=False, stop_words=[]):
#     words = []
#     for token in wordpunct_tokenize(re.sub('[%s]' % re.escape(string.punctuation), ' ', \
#             text.decode(encoding='UTF-8', errors='ignore'))):
#         if not token.isdigit() and not token in stop_words:
#             if stem:
#                 try:
#                     w = EnglishStemmer().stem(token)
#                 except Exception as e:
#                     w = token
#             else:
#                 w = token
#             words.append(w)
# 
#     return words
# 
#     # return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
#     #                     re.sub('[%s]' % re.escape(string.punctuation), ' ', text.decode(encoding='UTF-8', errors='ignore'))) if
#     #                     not token.isdigit() and not token in stop_words]
# 
# def get_all_files(corpus_path, recursive=False):
#     if recursive:
#         return [os.path.join(root, file) for root, dirnames, filenames in os.walk(corpus_path) for file in filenames if os.path.isfile(os.path.join(root, file)) and not file.startswith('.')]
#     else:
#         return [os.path.join(corpus_path, filename) for filename in os.listdir(corpus_path) if os.path.isfile(os.path.join(corpus_path, filename)) and not filename.startswith('.')]
# 
# def count_words(docs):
#     # count the number of times a word appears in a corpus
#     word_freq = defaultdict(lambda: 0)
#     for each in docs:
#         for word, val in each.iteritems():
#             word_freq[word] += val
# 
#     return word_freq
# 
# Path: autoencoder/datasets/reuters.py
# def construct_corpus(doc_word_freq, word_freq, training_phase, vocab_dict=None, threshold=5, topn=None):
#     if not (training_phase or isinstance(vocab_dict, dict)):
#         raise ValueError('vocab_dict must be provided if training_phase is set False')
# 
#     if training_phase:
#         vocab_dict = build_vocab(word_freq, threshold=threshold, topn=topn)
# 
#     docs = generate_bow(doc_word_freq, vocab_dict)
#     new_word_freq = dict([(vocab_dict[word], freq) for word, freq in word_freq.iteritems() if word in vocab_dict])
# 
#     return docs, vocab_dict, new_word_freq
# 
# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


import os
import re
import numpy as np
    import pdb;pdb.set_trace()
from random import shuffle
from collections import Counter
from ..preprocessing.preprocessing import init_stopwords, tiny_tokenize_xml, tiny_tokenize, get_all_files, count_words
from ..datasets.reuters import construct_corpus
from ..utils.io_utils import dump_json
        for filename in self.files:
            doc_name = os.path.basename(filename)
            if not doc_name in self.train_docs:
                continue
            try:
                with open(filename, 'r') as fp:
                    count += 1
                    text = fp.read().lower()
                    # remove punctuations, stopwords and *unnecessary digits*, stemming
                    words = tiny_tokenize(text, self.stem, cached_stop_words)
                    if self.with_docname:
                        yield [words, [doc_name]]
                    else:
                        yield words
            except Exception as e:
                raise e
        print count

def extract_contents(text, out_file):
    if not isinstance(text, unicode):
        text = text.decode('utf-8')
    contents = ' '.join(prog.findall(text))
    contents = tiny_tokenize_xml(contents, False, cached_stop_words)
    with open(out_file, 'w') as f:
        f.write(' '.join(contents))

    return contents

def xml2text(in_dir, out_dir, white_list=None):
    # it will be fast if white_list is a dict instead of a list

# Path: autoencoder/preprocessing/preprocessing.py
# def init_stopwords():
#     try:
#         stopword_path = 'patterns/english_stopwords.txt'
#         cached_stop_words = load_stopwords(os.path.join(os.path.split(__file__)[0], stopword_path))
#         print 'Loaded %s' % stopword_path
#     except:
#         from nltk.corpus import stopwords
#         cached_stop_words = stopwords.words("english")
#         print 'Loaded nltk.corpus.stopwords'
# 
#     return cached_stop_words
# 
# def tiny_tokenize_xml(text, stem=False, stop_words=[]):
#     return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
#                         re.sub('[%s]' % re.escape(string.punctuation), ' ', text.encode(encoding='ascii', errors='ignore'))) if
#                         not token.isdigit() and not token in stop_words]
# 
# def tiny_tokenize(text, stem=False, stop_words=[]):
#     words = []
#     for token in wordpunct_tokenize(re.sub('[%s]' % re.escape(string.punctuation), ' ', \
#             text.decode(encoding='UTF-8', errors='ignore'))):
#         if not token.isdigit() and not token in stop_words:
#             if stem:
#                 try:
#                     w = EnglishStemmer().stem(token)
#                 except Exception as e:
#                     w = token
#             else:
#                 w = token
#             words.append(w)
# 
#     return words
# 
#     # return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
#     #                     re.sub('[%s]' % re.escape(string.punctuation), ' ', text.decode(encoding='UTF-8', errors='ignore'))) if
#     #                     not token.isdigit() and not token in stop_words]
# 
# def get_all_files(corpus_path, recursive=False):
#     if recursive:
#         return [os.path.join(root, file) for root, dirnames, filenames in os.walk(corpus_path) for file in filenames if os.path.isfile(os.path.join(root, file)) and not file.startswith('.')]
#     else:
#         return [os.path.join(corpus_path, filename) for filename in os.listdir(corpus_path) if os.path.isfile(os.path.join(corpus_path, filename)) and not filename.startswith('.')]
# 
# def count_words(docs):
#     # count the number of times a word appears in a corpus
#     word_freq = defaultdict(lambda: 0)
#     for each in docs:
#         for word, val in each.iteritems():
#             word_freq[word] += val
# 
#     return word_freq
# 
# Path: autoencoder/datasets/reuters.py
# def construct_corpus(doc_word_freq, word_freq, training_phase, vocab_dict=None, threshold=5, topn=None):
#     if not (training_phase or isinstance(vocab_dict, dict)):
#         raise ValueError('vocab_dict must be provided if training_phase is set False')
# 
#     if training_phase:
#         vocab_dict = build_vocab(word_freq, threshold=threshold, topn=topn)
# 
#     docs = generate_bow(doc_word_freq, vocab_dict)
#     new_word_freq = dict([(vocab_dict[word], freq) for word, freq in word_freq.iteritems() if word in vocab_dict])
# 
#     return docs, vocab_dict, new_word_freq
# 
# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


# Path: autoencoder/datasets/wiki10plus.py
import os
import re
import numpy as np
    import pdb;pdb.set_trace()
from random import shuffle
from collections import Counter
from ..preprocessing.preprocessing import init_stopwords, tiny_tokenize_xml, tiny_tokenize, get_all_files, count_words
from ..datasets.reuters import construct_corpus
from ..utils.io_utils import dump_json

        for filename in self.files:
            doc_name = os.path.basename(filename)
            if not doc_name in self.train_docs:
                continue
            try:
                with open(filename, 'r') as fp:
                    count += 1
                    text = fp.read().lower()
                    # remove punctuations, stopwords and *unnecessary digits*, stemming
                    words = tiny_tokenize(text, self.stem, cached_stop_words)
                    if self.with_docname:
                        yield [words, [doc_name]]
                    else:
                        yield words
            except Exception as e:
                raise e
        print count

def extract_contents(text, out_file):
    if not isinstance(text, unicode):
        text = text.decode('utf-8')
    contents = ' '.join(prog.findall(text))
    contents = tiny_tokenize_xml(contents, False, cached_stop_words)
    with open(out_file, 'w') as f:
        f.write(' '.join(contents))

    return contents

def xml2text(in_dir, out_dir, white_list=None):
    # it will be fast if white_list is a dict instead of a list

    files = get_all_files(in_dir, recursive=False)