# Path: autoencoder/preprocessing/preprocessing.py
# def get_all_files(corpus_path, recursive=False):
#     if recursive:
#         return [os.path.join(root, file) for root, dirnames, filenames in os.walk(corpus_path) for file in filenames if os.path.isfile(os.path.join(root, file)) and not file.startswith('.')]
#     else:
#         return [os.path.join(corpus_path, filename) for filename in os.listdir(corpus_path) if os.path.isfile(os.path.join(corpus_path, filename)) and not filename.startswith('.')]
# 
# def init_stopwords():
#     try:
#         stopword_path = 'patterns/english_stopwords.txt'
#         cached_stop_words = load_stopwords(os.path.join(os.path.split(__file__)[0], stopword_path))
#         print 'Loaded %s' % stopword_path
#     except:
#         from nltk.corpus import stopwords
#         cached_stop_words = stopwords.words("english")
#         print 'Loaded nltk.corpus.stopwords'
# 
#     return cached_stop_words
# 
# def tiny_tokenize(text, stem=False, stop_words=[]):
#     words = []
#     for token in wordpunct_tokenize(re.sub('[%s]' % re.escape(string.punctuation), ' ', \
#             text.decode(encoding='UTF-8', errors='ignore'))):
#         if not token.isdigit() and not token in stop_words:
#             if stem:
#                 try:
#                     w = EnglishStemmer().stem(token)
#                 except Exception as e:
#                     w = token
#             else:
#                 w = token
#             words.append(w)
# 
#     return words
# 
#     # return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
#     #                     re.sub('[%s]' % re.escape(string.punctuation), ' ', text.decode(encoding='UTF-8', errors='ignore'))) if
#     #                     not token.isdigit() and not token in stop_words]


import os
from random import shuffle
from collections import defaultdict
from ..preprocessing.preprocessing import get_all_files, init_stopwords, tiny_tokenize
'''
Created on Jan, 2017

@author: hugo

'''
from __future__ import absolute_import



cached_stop_words = init_stopwords()


class CorpusIter20News(object):
    def __init__(self, corpus_path, recursive=False, stem=True, with_docname=False):
        self.stem = stem
        self.with_docname = with_docname

# Path: autoencoder/preprocessing/preprocessing.py
# def get_all_files(corpus_path, recursive=False):
#     if recursive:
#         return [os.path.join(root, file) for root, dirnames, filenames in os.walk(corpus_path) for file in filenames if os.path.isfile(os.path.join(root, file)) and not file.startswith('.')]
#     else:
#         return [os.path.join(corpus_path, filename) for filename in os.listdir(corpus_path) if os.path.isfile(os.path.join(corpus_path, filename)) and not filename.startswith('.')]
# 
# def init_stopwords():
#     try:
#         stopword_path = 'patterns/english_stopwords.txt'
#         cached_stop_words = load_stopwords(os.path.join(os.path.split(__file__)[0], stopword_path))
#         print 'Loaded %s' % stopword_path
#     except:
#         from nltk.corpus import stopwords
#         cached_stop_words = stopwords.words("english")
#         print 'Loaded nltk.corpus.stopwords'
# 
#     return cached_stop_words
# 
# def tiny_tokenize(text, stem=False, stop_words=[]):
#     words = []
#     for token in wordpunct_tokenize(re.sub('[%s]' % re.escape(string.punctuation), ' ', \
#             text.decode(encoding='UTF-8', errors='ignore'))):
#         if not token.isdigit() and not token in stop_words:
#             if stem:
#                 try:
#                     w = EnglishStemmer().stem(token)
#                 except Exception as e:
#                     w = token
#             else:
#                 w = token
#             words.append(w)
# 
#     return words
# 
#     # return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
#     #                     re.sub('[%s]' % re.escape(string.punctuation), ' ', text.decode(encoding='UTF-8', errors='ignore'))) if
#     #                     not token.isdigit() and not token in stop_words]


# Path: autoencoder/datasets/the20news.py
import os
from random import shuffle
from collections import defaultdict
from ..preprocessing.preprocessing import get_all_files, init_stopwords, tiny_tokenize

'''
Created on Jan, 2017

@author: hugo

'''
from __future__ import absolute_import



cached_stop_words = init_stopwords()


class CorpusIter20News(object):
    def __init__(self, corpus_path, recursive=False, stem=True, with_docname=False):
        self.stem = stem
        self.with_docname = with_docname

        self.files = get_all_files(corpus_path, recursive)