# Path: textblob_de/packages.py
# HERE = os.path.dirname(os.path.abspath(__file__))
# 
# Path: textblob_de/compat.py
# PY2 = int(sys.version[0]) == 2
# PY26 = PY2 and int(sys.version_info[1]) < 7
#     def implements_to_string(cls):
# def with_metaclass(meta, *bases):
#         def __new__(cls, name, this_bases, d):
# def decode_string(v, encoding="utf-8"):
# def encode_string(v, encoding="utf-8"):
# def get_external_executable(_exec):
# def _shutil_which(cmd, mode=os.F_OK | os.X_OK, path=None):
#     def _access_check(fn, mode):
#     class metaclass(meta):
# 
# Path: textblob_de/tokenizers.py
# class PatternTokenizer(BaseTokenizer):
# 
#     """Tokenizer included in ``pattern.de`` package.
# 
#     **PROs:**
# 
#     * handling of emoticons
#     * flexible implementations of abbreviations
#     * can be adapted very easily
# 
#     **CONs:**
# 
#     * ordinal numbers cause sentence breaks
#     * indices of Sentence() objects cannot be computed
# 
#     """
# 
#     def __init__(self):
#         self.tokens = []
# 
#     def tokenize(self, text, include_punc=True, nested=False):
#         """Return a list of word tokens.
# 
#         :param text: string of text.
#         :param include_punc: (optional) whether to include punctuation as separate
#             tokens. Default to True.
# 
#         """
#         self.tokens = [
#             w for w in (
#                 self.word_tokenize(
#                     s,
#                     include_punc) for s in self.sent_tokenize(text))]
#         if nested:
#             return self.tokens
#         else:
#             return list(chain.from_iterable(self.tokens))
# 
#     def sent_tokenize(self, text, **kwargs):
#         """Returns a list of sentences.
# 
#         Each sentence is a space-separated string of tokens (words).
#         Handles common cases of abbreviations (e.g., etc., ...).
#         Punctuation marks are split from other words. Periods (or ?!) mark the end of a sentence.
#         Headings without an ending period are inferred by line breaks.
# 
#         """
# 
#         sentences = find_sentences(text,
#                                    punctuation=kwargs.get(
#                                        "punctuation",
#                                        PUNCTUATION),
#                                    abbreviations=kwargs.get(
#                                        "abbreviations",
#                                        ABBREVIATIONS_DE),
#                                    replace=kwargs.get("replace", replacements),
#                                    linebreak=r"\n{2,}")
#         return sentences
# 
#     def word_tokenize(self, sentences, include_punc=True):
#         #: Do not process empty strings (Issue #3)
#         if sentences.strip() == "":
#             return []
#         _tokens = sentences.split(" ")
#         #: Handle strings consisting of a single punctuation mark seperately (Issue #4)
#         if len(_tokens) == 1:
#             if _tokens[0] in PUNCTUATION:
#                 if include_punc:
#                     return _tokens
#                 else:
#                     return []
#         if include_punc:
#             last_word = _tokens[-1]
#             # Make sure that you do not separate '.' tokens into ['', '.']
#             # (Issue #5)
#             if last_word.endswith('.') and len(last_word) > 1:
#                 _tokens = _tokens[:-1] + [last_word[:-1], '.']
#             return _tokens
#         else:
#             # Return each word token
#             # Strips punctuation unless the word comes from a contraction
#             # e.g. "gibt's" => ["gibt", "'s"] in "Heute gibt's viel zu tun!"
#             # e.g. "hat's" => ["hat", "'s"]
#             # e.g. "home." => ['home']
#             words = [
#                 word if word.startswith("'") else strip_punc(
#                     word,
#                     all=False) for word in _tokens if strip_punc(
#                     word,
#                     all=False)]
#             return list(words)


import string
from textblob.base import BaseTagger
from textblob.utils import PUNCTUATION_REGEX
from textblob_de.packages import pattern_de
from textblob_de.compat import unicode
from textblob_de.tokenizers import PatternTokenizer
            (space separated string).

        """
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(

# Path: textblob_de/packages.py
# HERE = os.path.dirname(os.path.abspath(__file__))
# 
# Path: textblob_de/compat.py
# PY2 = int(sys.version[0]) == 2
# PY26 = PY2 and int(sys.version_info[1]) < 7
#     def implements_to_string(cls):
# def with_metaclass(meta, *bases):
#         def __new__(cls, name, this_bases, d):
# def decode_string(v, encoding="utf-8"):
# def encode_string(v, encoding="utf-8"):
# def get_external_executable(_exec):
# def _shutil_which(cmd, mode=os.F_OK | os.X_OK, path=None):
#     def _access_check(fn, mode):
#     class metaclass(meta):
# 
# Path: textblob_de/tokenizers.py
# class PatternTokenizer(BaseTokenizer):
# 
#     """Tokenizer included in ``pattern.de`` package.
# 
#     **PROs:**
# 
#     * handling of emoticons
#     * flexible implementations of abbreviations
#     * can be adapted very easily
# 
#     **CONs:**
# 
#     * ordinal numbers cause sentence breaks
#     * indices of Sentence() objects cannot be computed
# 
#     """
# 
#     def __init__(self):
#         self.tokens = []
# 
#     def tokenize(self, text, include_punc=True, nested=False):
#         """Return a list of word tokens.
# 
#         :param text: string of text.
#         :param include_punc: (optional) whether to include punctuation as separate
#             tokens. Default to True.
# 
#         """
#         self.tokens = [
#             w for w in (
#                 self.word_tokenize(
#                     s,
#                     include_punc) for s in self.sent_tokenize(text))]
#         if nested:
#             return self.tokens
#         else:
#             return list(chain.from_iterable(self.tokens))
# 
#     def sent_tokenize(self, text, **kwargs):
#         """Returns a list of sentences.
# 
#         Each sentence is a space-separated string of tokens (words).
#         Handles common cases of abbreviations (e.g., etc., ...).
#         Punctuation marks are split from other words. Periods (or ?!) mark the end of a sentence.
#         Headings without an ending period are inferred by line breaks.
# 
#         """
# 
#         sentences = find_sentences(text,
#                                    punctuation=kwargs.get(
#                                        "punctuation",
#                                        PUNCTUATION),
#                                    abbreviations=kwargs.get(
#                                        "abbreviations",
#                                        ABBREVIATIONS_DE),
#                                    replace=kwargs.get("replace", replacements),
#                                    linebreak=r"\n{2,}")
#         return sentences
# 
#     def word_tokenize(self, sentences, include_punc=True):
#         #: Do not process empty strings (Issue #3)
#         if sentences.strip() == "":
#             return []
#         _tokens = sentences.split(" ")
#         #: Handle strings consisting of a single punctuation mark seperately (Issue #4)
#         if len(_tokens) == 1:
#             if _tokens[0] in PUNCTUATION:
#                 if include_punc:
#                     return _tokens
#                 else:
#                     return []
#         if include_punc:
#             last_word = _tokens[-1]
#             # Make sure that you do not separate '.' tokens into ['', '.']
#             # (Issue #5)
#             if last_word.endswith('.') and len(last_word) > 1:
#                 _tokens = _tokens[:-1] + [last_word[:-1], '.']
#             return _tokens
#         else:
#             # Return each word token
#             # Strips punctuation unless the word comes from a contraction
#             # e.g. "gibt's" => ["gibt", "'s"] in "Heute gibt's viel zu tun!"
#             # e.g. "hat's" => ["hat", "'s"]
#             # e.g. "home." => ['home']
#             words = [
#                 word if word.startswith("'") else strip_punc(
#                     word,
#                     all=False) for word in _tokens if strip_punc(
#                     word,
#                     all=False)]
#             return list(words)


# Path: textblob_de/taggers.py
import string
from textblob.base import BaseTagger
from textblob.utils import PUNCTUATION_REGEX
from textblob_de.packages import pattern_de
from textblob_de.compat import unicode
from textblob_de.tokenizers import PatternTokenizer

            (space separated string).

        """
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(

                    unicode(t))]