parameters: (self, url)
return_statement: return result
docstring: Get the HTML for an URL, possibly from an in-memory cache.
  
          XXX TODO Note: this cache is never actually cleared. It's assumed that
          the data won't get stale over the lifetime of a locator instance (not
          necessarily true for the default_locator).
docstring_summary: Get the HTML for an URL, possibly from an in-memory cache.
docstring_tokens: Get the HTML for an URL possibly from an in - memory cache .
function: def get_page(self, url):
          """
          Get the HTML for an URL, possibly from an in-memory cache.
  
          XXX TODO Note: this cache is never actually cleared. It's assumed that
          the data won't get stale over the lifetime of a locator instance (not
          necessarily true for the default_locator).
          """
          # http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api
          scheme, netloc, path, _, _, _ = urlparse(url)
          if scheme == 'file' and os.path.isdir(url2pathname(path)):
              url = urljoin(ensure_slash(url), 'index.html')
  
          if url in self._page_cache:
              result = self._page_cache[url]
              logger.debug('Returning %s from cache: %s', url, result)
          else:
              host = netloc.split(':', 1)[0]
              result = None
              if host in self._bad_hosts:
                  logger.debug('Skipping %s due to bad host %s', url, host)
              else:
                  req = Request(url, headers={'Accept-encoding': 'identity'})
                  try:
                      logger.debug('Fetching %s', url)
                      resp = self.opener.open(req, timeout=self.timeout)
                      logger.debug('Fetched %s', url)
                      headers = resp.info()
                      content_type = headers.get('Content-Type', '')
                      if HTML_CONTENT_TYPE.match(content_type):
                          final_url = resp.geturl()
                          data = resp.read()
                          encoding = headers.get('Content-Encoding')
                          if encoding:
                              decoder = self.decoders[encoding]   # fail if not found
                              data = decoder(data)
                          encoding = 'utf-8'
                          m = CHARSET.search(content_type)
                          if m:
                              encoding = m.group(1)
                          try:
                              data = data.decode(encoding)
                          except UnicodeError:  # pragma: no cover
                              data = data.decode('latin-1')    # fallback
                          result = Page(data, final_url)
                          self._page_cache[final_url] = result
                  except HTTPError as e:
                      if e.code != 404:
                          logger.exception('Fetch failed: %s: %s', url, e)
                  except URLError as e:  # pragma: no cover
                      logger.exception('Fetch failed: %s: %s', url, e)
                      with self._lock:
                          self._bad_hosts.add(host)
                  except Exception as e:  # pragma: no cover
                      logger.exception('Fetch failed: %s: %s', url, e)
                  finally:
                      self._page_cache[url] = result   # even if None (failure)
          return result
function_tokens: def get_page ( self , url ) : # http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api scheme , netloc , path , _ , _ , _ = urlparse ( url ) if scheme == 'file' and os . path . isdir ( url2pathname ( path ) ) : url = urljoin ( ensure_slash ( url ) , 'index.html' ) if url in self . _page_cache : result = self . _page_cache [ url ] logger . debug ( 'Returning %s from cache: %s' , url , result ) else : host = netloc . split ( ':' , 1 ) [ 0 ] result = None if host in self . _bad_hosts : logger . debug ( 'Skipping %s due to bad host %s' , url , host ) else : req = Request ( url , headers = { 'Accept-encoding' : 'identity' } ) try : logger . debug ( 'Fetching %s' , url ) resp = self . opener . open ( req , timeout = self . timeout ) logger . debug ( 'Fetched %s' , url ) headers = resp . info ( ) content_type = headers . get ( 'Content-Type' , '' ) if HTML_CONTENT_TYPE . match ( content_type ) : final_url = resp . geturl ( ) data = resp . read ( ) encoding = headers . get ( 'Content-Encoding' ) if encoding : decoder = self . decoders [ encoding ] # fail if not found data = decoder ( data ) encoding = 'utf-8' m = CHARSET . search ( content_type ) if m : encoding = m . group ( 1 ) try : data = data . decode ( encoding ) except UnicodeError : # pragma: no cover data = data . decode ( 'latin-1' ) # fallback result = Page ( data , final_url ) self . _page_cache [ final_url ] = result except HTTPError as e : if e . code != 404 : logger . exception ( 'Fetch failed: %s: %s' , url , e ) except URLError as e : # pragma: no cover logger . exception ( 'Fetch failed: %s: %s' , url , e ) with self . _lock : self . _bad_hosts . add ( host ) except Exception as e : # pragma: no cover logger . exception ( 'Fetch failed: %s: %s' , url , e ) finally : self . _page_cache [ url ] = result # even if None (failure) return result