parameters: (tu, extent)
docstring: Helper method to return all tokens in an extent.
  
          This functionality is needed multiple places in this module. We define
          it here because it seems like a logical place.
docstring_summary: Helper method to return all tokens in an extent.
docstring_tokens: Helper method to return all tokens in an extent .
function: def get_tokens(tu, extent):
          """Helper method to return all tokens in an extent.
  
          This functionality is needed multiple places in this module. We define
          it here because it seems like a logical place.
          """
          tokens_memory = POINTER(Token)()
          tokens_count = c_uint()
  
          conf.lib.clang_tokenize(tu, extent, byref(tokens_memory),
                  byref(tokens_count))
  
          count = int(tokens_count.value)
  
          # If we get no tokens, no memory was allocated. Be sure not to return
          # anything and potentially call a destructor on nothing.
          if count < 1:
              return
  
          tokens_array = cast(tokens_memory, POINTER(Token * count)).contents
  
          token_group = TokenGroup(tu, tokens_memory, tokens_count)
  
          for i in xrange(0, count):
              token = Token()
              token.int_data = tokens_array[i].int_data
              token.ptr_data = tokens_array[i].ptr_data
              token._tu = tu
              token._group = token_group
  
              yield token
function_tokens: def get_tokens ( tu , extent ) : tokens_memory = POINTER ( Token ) ( ) tokens_count = c_uint ( ) conf . lib . clang_tokenize ( tu , extent , byref ( tokens_memory ) , byref ( tokens_count ) ) count = int ( tokens_count . value ) # If we get no tokens, no memory was allocated. Be sure not to return # anything and potentially call a destructor on nothing. if count < 1 : return tokens_array = cast ( tokens_memory , POINTER ( Token * count ) ) . contents token_group = TokenGroup ( tu , tokens_memory , tokens_count ) for i in xrange ( 0 , count ) : token = Token ( ) token . int_data = tokens_array [ i ] . int_data token . ptr_data = tokens_array [ i ] . ptr_data token . _tu = tu token . _group = token_group yield token