# Path: twip/util.py
# class Tokenizer(object):
# class FileLineGen(object):
# class FileBOWGen(object):
#     def __init__(self, doc=None, regex=CRE_TOKEN, strip=True, nonwords=False, nonwords_set=None, nonwords_regex=RE_NONWORD,
#                  lower=None, stem=None, ngram_delim=' ', ngrams=1):
#     def __call__(self, doc):
#     def __reduce__(self):
#     def span_tokenize(self, s):
#     def tokenize_sents(self, strings):
#     def span_tokenize_sents(self, strings):
#     def __iter__(self, ngrams=None):
#     def strip(self, s):
#     def stem(self, s):
#     def lemmatize(self, s):
#     def __getstate__(self):
#     def __setstate__(self, d):
# def make_tokenizer(tokenizer=Tokenizer, stem=False, strip=None, nonwords=None, lower=None):
# def compile_vocab(docs, limit=1e6, verbose=0, tokenizer=Tokenizer(stem=None, lower=None, strip=None)):
# def gen_bows(docs=None, vocab=None, limit=1e6, verbose=1, tokenizer=Tokenizer):
# def gen_file_lines(path, mode='rUb', strip_eol=True, ascii=True, eol='\n'):
#     def __init__(self, path, mode='rUb', eol='\n'):
#     def __iter__(self):
#     def __init__(self, path=None, verbose=0):
#     def __iter__(self):
# def interleave_skip(iterables, limit=None):
# 
# Path: twip/nlp.py
# def make_named_stemmer(stem=None, min_len=3):
#     """Construct a callable object and a string sufficient to reconstruct it later (unpickling)
# 
#     >>> make_named_stemmer('str_lower')
#     ('str_lower', <function str_lower at ...>)
#     >>> make_named_stemmer('Lancaster')
#     ('lancaster', <Stemmer object at ...>)
#     """
#     name, stem = stringify(stem), make_stemmer(stem=stem, min_len=min_len)
#     if hasattr(stem, '__name__'):
#         return stem.__name__, stem
#     if name.strip().lower() in STEMMER_TYPES:
#         return name.strip().lower(), stem
#     if hasattr(stem, 'pattern'):
#         return stem.pattern, stem
#     return stringify(stem), stem


from past.builtins import basestring
from pug.nlp.regex import RE_TOKEN, RE_NONWORD
from twip.util import str_strip, str_lower, to_ascii, stringify
from twip.nlp import make_named_stemmer
import re
        strip = strip or None
        # strip whitespace, overrides strip() method
        self.strip = strip if callable(strip) else (str_strip if strip else None)
        self.doc = to_ascii(doc)
        self.regex = regex
        if isinstance(self.regex, basestring):
            self.regex = re.compile(self.regex)
        self.nonwords = nonwords  # whether to use the default REGEX for nonwords
        self.nonwords_set = nonwords_set or set()
        self.nonwords_regex = nonwords_regex
        self.lower = lower if callable(lower) else (str_lower if lower else None)
        self.stemmer_name, self.stem = make_named_stemmer(stem)  # stem can be a callable Stemmer instance or just a function
        self.ngrams = ngrams or 1  # ngram degree, numger of ngrams per token
        if isinstance(self.nonwords_regex, basestring):
            self.nonwords_regex = re.compile(self.nonwords_regex)
        elif self.nonwords:
            try:
                self.nonwords_set = set(self.nonwords)
            except TypeError:
                self.nonwords_set = set(['None', 'none', 'and', 'but'])
                # if a set of nonwords has been provided dont use the internal nonwords REGEX?
                self.nonwords = not bool(self.nonwords)

    def __call__(self, doc):
        """Lazily tokenize a new document (tokens aren't generated until the class instance is iterated)

        >>> list(Tokenizer()('new string to parse'))
        ['new', 'string', 'to', 'parse']
        """
        # tokenization doesn't happen until you try to iterate through the Tokenizer instance or class

# Path: twip/util.py
# class Tokenizer(object):
# class FileLineGen(object):
# class FileBOWGen(object):
#     def __init__(self, doc=None, regex=CRE_TOKEN, strip=True, nonwords=False, nonwords_set=None, nonwords_regex=RE_NONWORD,
#                  lower=None, stem=None, ngram_delim=' ', ngrams=1):
#     def __call__(self, doc):
#     def __reduce__(self):
#     def span_tokenize(self, s):
#     def tokenize_sents(self, strings):
#     def span_tokenize_sents(self, strings):
#     def __iter__(self, ngrams=None):
#     def strip(self, s):
#     def stem(self, s):
#     def lemmatize(self, s):
#     def __getstate__(self):
#     def __setstate__(self, d):
# def make_tokenizer(tokenizer=Tokenizer, stem=False, strip=None, nonwords=None, lower=None):
# def compile_vocab(docs, limit=1e6, verbose=0, tokenizer=Tokenizer(stem=None, lower=None, strip=None)):
# def gen_bows(docs=None, vocab=None, limit=1e6, verbose=1, tokenizer=Tokenizer):
# def gen_file_lines(path, mode='rUb', strip_eol=True, ascii=True, eol='\n'):
#     def __init__(self, path, mode='rUb', eol='\n'):
#     def __iter__(self):
#     def __init__(self, path=None, verbose=0):
#     def __iter__(self):
# def interleave_skip(iterables, limit=None):
# 
# Path: twip/nlp.py
# def make_named_stemmer(stem=None, min_len=3):
#     """Construct a callable object and a string sufficient to reconstruct it later (unpickling)
# 
#     >>> make_named_stemmer('str_lower')
#     ('str_lower', <function str_lower at ...>)
#     >>> make_named_stemmer('Lancaster')
#     ('lancaster', <Stemmer object at ...>)
#     """
#     name, stem = stringify(stem), make_stemmer(stem=stem, min_len=min_len)
#     if hasattr(stem, '__name__'):
#         return stem.__name__, stem
#     if name.strip().lower() in STEMMER_TYPES:
#         return name.strip().lower(), stem
#     if hasattr(stem, 'pattern'):
#         return stem.pattern, stem
#     return stringify(stem), stem


# Path: twip/train.py
from past.builtins import basestring
from pug.nlp.regex import RE_TOKEN, RE_NONWORD
from twip.util import str_strip, str_lower, to_ascii, stringify
from twip.nlp import make_named_stemmer
import re

        strip = strip or None
        # strip whitespace, overrides strip() method
        self.strip = strip if callable(strip) else (str_strip if strip else None)
        self.doc = to_ascii(doc)
        self.regex = regex
        if isinstance(self.regex, basestring):
            self.regex = re.compile(self.regex)
        self.nonwords = nonwords  # whether to use the default REGEX for nonwords
        self.nonwords_set = nonwords_set or set()
        self.nonwords_regex = nonwords_regex
        self.lower = lower if callable(lower) else (str_lower if lower else None)
        self.stemmer_name, self.stem = make_named_stemmer(stem)  # stem can be a callable Stemmer instance or just a function
        self.ngrams = ngrams or 1  # ngram degree, numger of ngrams per token
        if isinstance(self.nonwords_regex, basestring):
            self.nonwords_regex = re.compile(self.nonwords_regex)
        elif self.nonwords:
            try:
                self.nonwords_set = set(self.nonwords)
            except TypeError:
                self.nonwords_set = set(['None', 'none', 'and', 'but'])
                # if a set of nonwords has been provided dont use the internal nonwords REGEX?
                self.nonwords = not bool(self.nonwords)

    def __call__(self, doc):
        """Lazily tokenize a new document (tokens aren't generated until the class instance is iterated)

        >>> list(Tokenizer()('new string to parse'))
        ['new', 'string', 'to', 'parse']
        """
        # tokenization doesn't happen until you try to iterate through the Tokenizer instance or class

        self.doc = to_ascii(doc)