parameters: (string)
return_statement: return res
docstring: Split string into tokens and return them as a list.
docstring_summary: Split string into tokens and return them as a list.
docstring_tokens: Split string into tokens and return them as a list .
function: def tokenize(string):
      """Split string into tokens and return them as a list."""
      mitie_tokenize = _f.mitie_tokenize
      mitie_tokenize.restype = ctypes.POINTER(ctypes.c_char_p)
      mitie_tokenize.argtypes = ctypes.c_char_p,
      string = to_bytes(string)
      tok = mitie_tokenize(string)
      if tok is None:
          raise Exception("Unable to tokenize string.")
      i = 0
      res = []
      while tok[i] is not None:
          res.append(tok[i])
          i += 1
      _f.mitie_free(tok)
      return res
function_tokens: def tokenize ( string ) : mitie_tokenize = _f . mitie_tokenize mitie_tokenize . restype = ctypes . POINTER ( ctypes . c_char_p ) mitie_tokenize . argtypes = ctypes . c_char_p , string = to_bytes ( string ) tok = mitie_tokenize ( string ) if tok is None : raise Exception ( "Unable to tokenize string." ) i = 0 res = [ ] while tok [ i ] is not None : res . append ( tok [ i ] ) i += 1 _f . mitie_free ( tok ) return res