if href[0] == '/':
    href = (domain_link + href).strip()
elif href[:4] == 'http':
    href = href.strip()
elif href[0] != '/' and href[:4] != 'http':
    href = (domain_link + '/' + href).strip()
if '#' in href:
    indx = href.index('#')
href = href[:indx].strip()
if href in links:
    continue
links.append(self.re_encode(href))
return links
Beautiful Soup open all the url with pid in it