# Path: textblob_de/tokenizers.py
# def word_tokenize(self, text, include_punc=True):
#     """The Treebank tokenizer uses regular expressions to tokenize text as
#     in Penn Treebank.
# 
#     It assumes that the text has already been segmented into sentences,
#     e.g. using ``self.sent_tokenize()``.
# 
#     This tokenizer performs the following steps:
# 
#     - split standard contractions, e.g. ``don't`` -> ``do n't`` and ``they'll`` -> ``they 'll``
#     - treat most punctuation characters as separate tokens
#     - split off commas and single quotes, when followed by whitespace
#     - separate periods that appear at the end of line
# 
#     Source: NLTK's docstring of ``TreebankWordTokenizer`` (accessed: 02/10/2014)
# 
#     """
#     #: Do not process empty strings (Issue #3)
#     if text.strip() == "":
#         return []
#     _tokens = self.word_tok.tokenize(text)
#     #: Handle strings consisting of a single punctuation mark seperately (Issue #4)
#     if len(_tokens) == 1:
#         if _tokens[0] in PUNCTUATION:
#             if include_punc:
#                 return _tokens
#             else:
#                 return []
#     if include_punc:
#         return _tokens
#     else:
#         # Return each word token
#         # Strips punctuation unless the word comes from a contraction
#         # e.g. "gibt's" => ["gibt", "'s"] in "Heute gibt's viel zu tun!"
#         # e.g. "hat's" => ["hat", "'s"]
#         # e.g. "home." => ['home']
#         words = [
#             word if word.startswith("'") else strip_punc(
#                 word,
#                 all=False) for word in _tokens if strip_punc(
#                 word,
#                 all=False)]
#         return list(words)
# 
# Path: textblob_de/compat.py
# PY2 = int(sys.version[0]) == 2
# PY26 = PY2 and int(sys.version_info[1]) < 7
#     def implements_to_string(cls):
# def with_metaclass(meta, *bases):
#         def __new__(cls, name, this_bases, d):
# def decode_string(v, encoding="utf-8"):
# def encode_string(v, encoding="utf-8"):
# def get_external_executable(_exec):
# def _shutil_which(cmd, mode=os.F_OK | os.X_OK, path=None):
#     def _access_check(fn, mode):
#     class metaclass(meta):


from itertools import chain
from textblob.utils import strip_punc
from textblob.decorators import cached_property
from textblob_de.tokenizers import word_tokenize
from textblob_de.compat import basestring
import nltk
import textblob.formats as formats

"""
from __future__ import absolute_import



### Basic feature extractors ###


def _get_words_from_dataset(dataset):
    """Return a set of all words in a dataset.

    :param dataset: A list of tuples of the form ``(words, label)`` where
        ``words`` is either a string of a list of tokens.

    """
    # Words may be either a string or a list of tokens. Return an iterator
    # of tokens accordingly
    def tokenize(words):
        if isinstance(words, basestring):
            return word_tokenize(words, include_punc=False)
        else:
            return words
    all_words = chain.from_iterable(tokenize(words) for words, _ in dataset)
    return set(all_words)


def _get_document_tokens(document):
    if isinstance(document, basestring):
        tokens = set((strip_punc(w, all=False)

# Path: textblob_de/tokenizers.py
# def word_tokenize(self, text, include_punc=True):
#     """The Treebank tokenizer uses regular expressions to tokenize text as
#     in Penn Treebank.
# 
#     It assumes that the text has already been segmented into sentences,
#     e.g. using ``self.sent_tokenize()``.
# 
#     This tokenizer performs the following steps:
# 
#     - split standard contractions, e.g. ``don't`` -> ``do n't`` and ``they'll`` -> ``they 'll``
#     - treat most punctuation characters as separate tokens
#     - split off commas and single quotes, when followed by whitespace
#     - separate periods that appear at the end of line
# 
#     Source: NLTK's docstring of ``TreebankWordTokenizer`` (accessed: 02/10/2014)
# 
#     """
#     #: Do not process empty strings (Issue #3)
#     if text.strip() == "":
#         return []
#     _tokens = self.word_tok.tokenize(text)
#     #: Handle strings consisting of a single punctuation mark seperately (Issue #4)
#     if len(_tokens) == 1:
#         if _tokens[0] in PUNCTUATION:
#             if include_punc:
#                 return _tokens
#             else:
#                 return []
#     if include_punc:
#         return _tokens
#     else:
#         # Return each word token
#         # Strips punctuation unless the word comes from a contraction
#         # e.g. "gibt's" => ["gibt", "'s"] in "Heute gibt's viel zu tun!"
#         # e.g. "hat's" => ["hat", "'s"]
#         # e.g. "home." => ['home']
#         words = [
#             word if word.startswith("'") else strip_punc(
#                 word,
#                 all=False) for word in _tokens if strip_punc(
#                 word,
#                 all=False)]
#         return list(words)
# 
# Path: textblob_de/compat.py
# PY2 = int(sys.version[0]) == 2
# PY26 = PY2 and int(sys.version_info[1]) < 7
#     def implements_to_string(cls):
# def with_metaclass(meta, *bases):
#         def __new__(cls, name, this_bases, d):
# def decode_string(v, encoding="utf-8"):
# def encode_string(v, encoding="utf-8"):
# def get_external_executable(_exec):
# def _shutil_which(cmd, mode=os.F_OK | os.X_OK, path=None):
#     def _access_check(fn, mode):
#     class metaclass(meta):


# Path: textblob_de/classifiers.py
from itertools import chain
from textblob.utils import strip_punc
from textblob.decorators import cached_property
from textblob_de.tokenizers import word_tokenize
from textblob_de.compat import basestring
import nltk
import textblob.formats as formats


"""
from __future__ import absolute_import



### Basic feature extractors ###


def _get_words_from_dataset(dataset):
    """Return a set of all words in a dataset.

    :param dataset: A list of tuples of the form ``(words, label)`` where
        ``words`` is either a string of a list of tokens.

    """
    # Words may be either a string or a list of tokens. Return an iterator
    # of tokens accordingly
    def tokenize(words):
        if isinstance(words, basestring):
            return word_tokenize(words, include_punc=False)
        else:
            return words
    all_words = chain.from_iterable(tokenize(words) for words, _ in dataset)
    return set(all_words)


def _get_document_tokens(document):
    if isinstance(document, basestring):
        tokens = set((strip_punc(w, all=False)

                      for w in word_tokenize(document, include_punc=False)))