parameters: (self)
docstring: Return a function to preprocess the text before tokenization
docstring_summary: Return a function to preprocess the text before tokenization
docstring_tokens: Return a function to preprocess the text before tokenization
function: def build_preprocessor(self):
          """Return a function to preprocess the text before tokenization"""
          if self.preprocessor is not None:
              return self.preprocessor
  
          # unfortunately python functools package does not have an efficient
          # `compose` function that would have allowed us to chain a dynamic
          # number of functions. However the cost of a lambda call is a few
          # hundreds of nanoseconds which is negligible when compared to the
          # cost of tokenizing a string of 1000 chars for instance.
          noop = lambda x: x
  
          # accent stripping
          if not self.strip_accents:
              strip_accents = noop
          elif callable(self.strip_accents):
              strip_accents = self.strip_accents
          elif self.strip_accents == 'ascii':
              strip_accents = strip_accents_ascii
          elif self.strip_accents == 'unicode':
              strip_accents = strip_accents_unicode
          else:
              raise ValueError('Invalid value for "strip_accents": %s' %
                               self.strip_accents)
  
          if self.lowercase:
              return lambda x: strip_accents(x.lower())
          else:
              return strip_accents
function_tokens: def build_preprocessor ( self ) : if self . preprocessor is not None : return self . preprocessor # unfortunately python functools package does not have an efficient # `compose` function that would have allowed us to chain a dynamic # number of functions. However the cost of a lambda call is a few # hundreds of nanoseconds which is negligible when compared to the # cost of tokenizing a string of 1000 chars for instance. noop = lambda x : x # accent stripping if not self . strip_accents : strip_accents = noop elif callable ( self . strip_accents ) : strip_accents = self . strip_accents elif self . strip_accents == 'ascii' : strip_accents = strip_accents_ascii elif self . strip_accents == 'unicode' : strip_accents = strip_accents_unicode else : raise ValueError ( 'Invalid value for "strip_accents": %s' % self . strip_accents ) if self . lowercase : return lambda x : strip_accents ( x . lower ( ) ) else : return strip_accents