parameters: (self, url, page)
docstring: Process the contents of a PyPI page
docstring_summary: Process the contents of a PyPI page
docstring_tokens: Process the contents of a PyPI page
function: def process_index(self, url, page):
          """Process the contents of a PyPI page"""
  
          def scan(link):
              # Process a URL to see if it's for a package page
              if link.startswith(self.index_url):
                  parts = list(map(
                      urllib.parse.unquote, link[len(self.index_url):].split('/')
                  ))
                  if len(parts) == 2 and '#' not in parts[1]:
                      # it's a package page, sanitize and index it
                      pkg = safe_name(parts[0])
                      ver = safe_version(parts[1])
                      self.package_pages.setdefault(pkg.lower(), {})[link] = True
                      return to_filename(pkg), to_filename(ver)
              return None, None
  
          # process an index page into the package-page index
          for match in HREF.finditer(page):
              try:
                  scan(urllib.parse.urljoin(url, htmldecode(match.group(1))))
              except ValueError:
                  pass
  
          pkg, ver = scan(url)  # ensure this page is in the page index
          if pkg:
              # process individual package page
              for new_url in find_external_links(url, page):
                  # Process the found URL
                  base, frag = egg_info_for_url(new_url)
                  if base.endswith('.py') and not frag:
                      if ver:
                          new_url += '#egg=%s-%s' % (pkg, ver)
                      else:
                          self.need_version_info(url)
                  self.scan_url(new_url)
  
              return PYPI_MD5.sub(
                  lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page
              )
          else:
              return ""
function_tokens: def process_index ( self , url , page ) : def scan ( link ) : # Process a URL to see if it's for a package page if link . startswith ( self . index_url ) : parts = list ( map ( urllib . parse . unquote , link [ len ( self . index_url ) : ] . split ( '/' ) ) ) if len ( parts ) == 2 and '#' not in parts [ 1 ] : # it's a package page, sanitize and index it pkg = safe_name ( parts [ 0 ] ) ver = safe_version ( parts [ 1 ] ) self . package_pages . setdefault ( pkg . lower ( ) , { } ) [ link ] = True return to_filename ( pkg ) , to_filename ( ver ) return None , None # process an index page into the package-page index for match in HREF . finditer ( page ) : try : scan ( urllib . parse . urljoin ( url , htmldecode ( match . group ( 1 ) ) ) ) except ValueError : pass pkg , ver = scan ( url ) # ensure this page is in the page index if pkg : # process individual package page for new_url in find_external_links ( url , page ) : # Process the found URL base , frag = egg_info_for_url ( new_url ) if base . endswith ( '.py' ) and not frag : if ver : new_url += '#egg=%s-%s' % ( pkg , ver ) else : self . need_version_info ( url ) self . scan_url ( new_url ) return PYPI_MD5 . sub ( lambda m : '<a href="%s#md5=%s">%s</a>' % m . group ( 1 , 3 , 2 ) , page ) else : return ""