parameters: (self, url, retrieve=False)
docstring: Evaluate a URL as a possible download, and maybe retrieve it
docstring_summary: Evaluate a URL as a possible download, and maybe retrieve it
docstring_tokens: Evaluate a URL as a possible download and maybe retrieve it
function: def process_url(self, url, retrieve=False):
          """Evaluate a URL as a possible download, and maybe retrieve it"""
          if url in self.scanned_urls and not retrieve:
              return
          self.scanned_urls[url] = True
          if not URL_SCHEME(url):
              self.process_filename(url)
              return
          else:
              dists = list(distros_for_url(url))
              if dists:
                  if not self.url_ok(url):
                      return
                  self.debug("Found link: %s", url)
  
          if dists or not retrieve or url in self.fetched_urls:
              list(map(self.add, dists))
              return  # don't need the actual page
  
          if not self.url_ok(url):
              self.fetched_urls[url] = True
              return
  
          self.info("Reading %s", url)
          self.fetched_urls[url] = True  # prevent multiple fetch attempts
          tmpl = "Download error on %s: %%s -- Some packages may not be found!"
          f = self.open_url(url, tmpl % url)
          if f is None:
              return
          self.fetched_urls[f.url] = True
          if 'html' not in f.headers.get('content-type', '').lower():
              f.close()  # not html, we can't process it
              return
  
          base = f.url  # handle redirects
          page = f.read()
          if not isinstance(page, str):
              # In Python 3 and got bytes but want str.
              if isinstance(f, urllib.error.HTTPError):
                  # Errors have no charset, assume latin1:
                  charset = 'latin-1'
              else:
                  charset = f.headers.get_param('charset') or 'latin-1'
              page = page.decode(charset, "ignore")
          f.close()
          for match in HREF.finditer(page):
              link = urllib.parse.urljoin(base, htmldecode(match.group(1)))
              self.process_url(link)
          if url.startswith(self.index_url) and getattr(f, 'code', None) != 404:
              page = self.process_index(url, page)
function_tokens: def process_url ( self , url , retrieve = False ) : if url in self . scanned_urls and not retrieve : return self . scanned_urls [ url ] = True if not URL_SCHEME ( url ) : self . process_filename ( url ) return else : dists = list ( distros_for_url ( url ) ) if dists : if not self . url_ok ( url ) : return self . debug ( "Found link: %s" , url ) if dists or not retrieve or url in self . fetched_urls : list ( map ( self . add , dists ) ) return # don't need the actual page if not self . url_ok ( url ) : self . fetched_urls [ url ] = True return self . info ( "Reading %s" , url ) self . fetched_urls [ url ] = True # prevent multiple fetch attempts tmpl = "Download error on %s: %%s -- Some packages may not be found!" f = self . open_url ( url , tmpl % url ) if f is None : return self . fetched_urls [ f . url ] = True if 'html' not in f . headers . get ( 'content-type' , '' ) . lower ( ) : f . close ( ) # not html, we can't process it return base = f . url # handle redirects page = f . read ( ) if not isinstance ( page , str ) : # In Python 3 and got bytes but want str. if isinstance ( f , urllib . error . HTTPError ) : # Errors have no charset, assume latin1: charset = 'latin-1' else : charset = f . headers . get_param ( 'charset' ) or 'latin-1' page = page . decode ( charset , "ignore" ) f . close ( ) for match in HREF . finditer ( page ) : link = urllib . parse . urljoin ( base , htmldecode ( match . group ( 1 ) ) ) self . process_url ( link ) if url . startswith ( self . index_url ) and getattr ( f , 'code' , None ) != 404 : page = self . process_index ( url , page )