# Path: twip/util.py
# class Tokenizer(object):
# class FileLineGen(object):
# class FileBOWGen(object):
#     def __init__(self, doc=None, regex=CRE_TOKEN, strip=True, nonwords=False, nonwords_set=None, nonwords_regex=RE_NONWORD,
#                  lower=None, stem=None, ngram_delim=' ', ngrams=1):
#     def __call__(self, doc):
#     def __reduce__(self):
#     def span_tokenize(self, s):
#     def tokenize_sents(self, strings):
#     def span_tokenize_sents(self, strings):
#     def __iter__(self, ngrams=None):
#     def strip(self, s):
#     def stem(self, s):
#     def lemmatize(self, s):
#     def __getstate__(self):
#     def __setstate__(self, d):
# def make_tokenizer(tokenizer=Tokenizer, stem=False, strip=None, nonwords=None, lower=None):
# def compile_vocab(docs, limit=1e6, verbose=0, tokenizer=Tokenizer(stem=None, lower=None, strip=None)):
# def gen_bows(docs=None, vocab=None, limit=1e6, verbose=1, tokenizer=Tokenizer):
# def gen_file_lines(path, mode='rUb', strip_eol=True, ascii=True, eol='\n'):
#     def __init__(self, path, mode='rUb', eol='\n'):
#     def __iter__(self):
#     def __init__(self, path=None, verbose=0):
#     def __iter__(self):
# def interleave_skip(iterables, limit=None):
# 
# Path: twip/nlp.py
# def make_named_stemmer(stem=None, min_len=3):
#     """Construct a callable object and a string sufficient to reconstruct it later (unpickling)
# 
#     >>> make_named_stemmer('str_lower')
#     ('str_lower', <function str_lower at ...>)
#     >>> make_named_stemmer('Lancaster')
#     ('lancaster', <Stemmer object at ...>)
#     """
#     name, stem = stringify(stem), make_stemmer(stem=stem, min_len=min_len)
#     if hasattr(stem, '__name__'):
#         return stem.__name__, stem
#     if name.strip().lower() in STEMMER_TYPES:
#         return name.strip().lower(), stem
#     if hasattr(stem, 'pattern'):
#         return stem.pattern, stem
#     return stringify(stem), stem


from past.builtins import basestring
from pug.nlp.regex import RE_TOKEN, RE_NONWORD
from twip.util import str_strip, str_lower, to_ascii, stringify
from twip.nlp import make_named_stemmer
import re
        The python splitter will produce 2 tokens that are only punctuation ("/")
        >>> len([s for s in doc.split() if s])
        72

        The built-in nonword REGEX ignores all-punctuation words, so there are 2 less here:
        >>> len(list(Tokenizer(doc, strip=False, nonwords=False)))
        70

        In addition, punctuation at the end of tokens is stripped so "D. Rock" doesn't tokenize to "D." but rather "D"
        >>> run_together_tokens = ''.join(list(Tokenizer(doc, strip=False, nonwords=False)))
        >>> '/' in run_together_tokens or ':' in ''.join(run_together_tokens)
        False

        But you can turn off stripping when instantiating the object.
        >>> all(t in Tokenizer(doc, strip=False, nonwords=True) for t in ('D', '_Project', 'Lead_', "90's", "product-delivery"))
        True
        """
        ngrams = ngrams or self.ngrams
        # FIXME: Improve memory efficiency by making this ngram tokenizer an actual generator
        if ngrams > 1:
            original_tokens = list(self.__iter__(ngrams=1))
            for tok in original_tokens:
                yield tok
            for i in range(2, ngrams + 1):
                for tok in list_ngrams(original_tokens, n=i, join=' '):
                    yield tok
        else:
            for w in self.regex.finditer(self.doc):
                if w:
                    w = w.group()

# Path: twip/util.py
# class Tokenizer(object):
# class FileLineGen(object):
# class FileBOWGen(object):
#     def __init__(self, doc=None, regex=CRE_TOKEN, strip=True, nonwords=False, nonwords_set=None, nonwords_regex=RE_NONWORD,
#                  lower=None, stem=None, ngram_delim=' ', ngrams=1):
#     def __call__(self, doc):
#     def __reduce__(self):
#     def span_tokenize(self, s):
#     def tokenize_sents(self, strings):
#     def span_tokenize_sents(self, strings):
#     def __iter__(self, ngrams=None):
#     def strip(self, s):
#     def stem(self, s):
#     def lemmatize(self, s):
#     def __getstate__(self):
#     def __setstate__(self, d):
# def make_tokenizer(tokenizer=Tokenizer, stem=False, strip=None, nonwords=None, lower=None):
# def compile_vocab(docs, limit=1e6, verbose=0, tokenizer=Tokenizer(stem=None, lower=None, strip=None)):
# def gen_bows(docs=None, vocab=None, limit=1e6, verbose=1, tokenizer=Tokenizer):
# def gen_file_lines(path, mode='rUb', strip_eol=True, ascii=True, eol='\n'):
#     def __init__(self, path, mode='rUb', eol='\n'):
#     def __iter__(self):
#     def __init__(self, path=None, verbose=0):
#     def __iter__(self):
# def interleave_skip(iterables, limit=None):
# 
# Path: twip/nlp.py
# def make_named_stemmer(stem=None, min_len=3):
#     """Construct a callable object and a string sufficient to reconstruct it later (unpickling)
# 
#     >>> make_named_stemmer('str_lower')
#     ('str_lower', <function str_lower at ...>)
#     >>> make_named_stemmer('Lancaster')
#     ('lancaster', <Stemmer object at ...>)
#     """
#     name, stem = stringify(stem), make_stemmer(stem=stem, min_len=min_len)
#     if hasattr(stem, '__name__'):
#         return stem.__name__, stem
#     if name.strip().lower() in STEMMER_TYPES:
#         return name.strip().lower(), stem
#     if hasattr(stem, 'pattern'):
#         return stem.pattern, stem
#     return stringify(stem), stem


# Path: twip/train.py
from past.builtins import basestring
from pug.nlp.regex import RE_TOKEN, RE_NONWORD
from twip.util import str_strip, str_lower, to_ascii, stringify
from twip.nlp import make_named_stemmer
import re

        The python splitter will produce 2 tokens that are only punctuation ("/")
        >>> len([s for s in doc.split() if s])
        72

        The built-in nonword REGEX ignores all-punctuation words, so there are 2 less here:
        >>> len(list(Tokenizer(doc, strip=False, nonwords=False)))
        70

        In addition, punctuation at the end of tokens is stripped so "D. Rock" doesn't tokenize to "D." but rather "D"
        >>> run_together_tokens = ''.join(list(Tokenizer(doc, strip=False, nonwords=False)))
        >>> '/' in run_together_tokens or ':' in ''.join(run_together_tokens)
        False

        But you can turn off stripping when instantiating the object.
        >>> all(t in Tokenizer(doc, strip=False, nonwords=True) for t in ('D', '_Project', 'Lead_', "90's", "product-delivery"))
        True
        """
        ngrams = ngrams or self.ngrams
        # FIXME: Improve memory efficiency by making this ngram tokenizer an actual generator
        if ngrams > 1:
            original_tokens = list(self.__iter__(ngrams=1))
            for tok in original_tokens:
                yield tok
            for i in range(2, ngrams + 1):
                for tok in list_ngrams(original_tokens, n=i, join=' '):
                    yield tok
        else:
            for w in self.regex.finditer(self.doc):
                if w:
                    w = w.group()

                    w = w if not self.strip_chars else str_strip(w, self.strip_chars)