def get_links(self):
    links = []
host = urlparse(self.url).hostname
scheme = urlparse(self.url).scheme
domain_link = scheme + '://' + host
for a in self.soup.find_all(href=True):
    href = a['href']
if not href or len(href) <= 1:
    continue
if href.lower().find('/pid/') != -1:
    if 'javascript:' in href.lower():
        continue
elif 'forgotpassword' in href.lower():
    continue
elif 'images' in href.lower():
    continue
elif 'seller-account' in href.lower():
    continue
elif 'review' in href.lower():
    continue
if href[0] == '/':
    href = (domain_link + href).strip()
elif href[:4] == 'http':
    href = href.strip()
elif href[0] != '/' and href[:4] != 'http':
    href = (domain_link + '/' + href).strip()
if '#' in href:
    indx = href.index('#')
href = href[:indx].strip()
if href in links:
    continue
links.append(self.re_encode(href))
return links
Beautiful Soup open all the url with pid in it