I'm very new to python and only vaguely remember OOP from doing some Java a few years ago so I don't know what the best way to do this is.I've build a bunch of classes that represent a crawler that scrapes images from a specific website. e.g: for the website stocksnap I have a class StocksnapCrawlerI have 9 of these crawler classes and it's awful, I know there can be a much better way of representing them, they share a lot in common.here are three of these crawlers:class MagdeleineCrawler:    def __init__(self, crawler_db):        self.current_page = crawler_db.current_page        self.crawler_db = crawler_db    def crawl(self):        current_page = self.current_page        print(Starting crawl on page  + str(current_page))        while True:            print(crawling page  + str(current_page))            page_response = requests.get(                'http://magdeleine.co/license/cc0/page/{}/'.format(current_page))            page_soup = BeautifulSoup(page_response.text)            image_links = [link[href]                           for link in page_soup.find_all('a', {'class': 'photo-link'})]            for image_link in image_links:                print(scraping image at  + image_link)                response = requests.get(image_link)                image_page_soup = BeautifulSoup(response.text)                print('getting image source link')                image_source_link = image_page_soup.find(                    'a', {'class': 'download'})['href']                # Get Tags                print('getting tags')                ul = image_page_soup.find('ul', {'class': 'tags'})                if ul:                    tag_links = ul.find_all('a', {'rel': 'tag'})                    tag_names = [tag_link.string for tag_link in tag_links]                    try:                        tag_names.remove('editor\'s pick')                    except:                        pass                thumbnail_url = image_page_soup.find(                    'img', {'id': 'main-img'})['src']                print(storing image in db)                store_image(                    image_source_link, image_link, thumbnail_url, 'MG', tag_names)                self.crawler_db.images_scraped += 1                self.crawler_db.save()            current_page += 1            self.crawler_db.current_page += 1            self.crawler_db.save()class FancycraveCrawler:    def __init__(self, crawler_db):        self.current_page = crawler_db.current_page        self.crawler_db = crawler_db    def crawl(self):        current_page = self.current_page        print(Starting crawl on page  + str(current_page))        while True:            print(crawling page  + str(current_page))            page_response = requests.get(                'http://fancycrave.com/page/{}'.format(current_page))            page_soup = BeautifulSoup(page_response.text)            image_articles = page_soup.find_all(                'article', {'class': 'type-photo'})            for image_article in image_articles:                print(scraping image)                image_source_link = image_article.find(                    'a', text='Download')['href']                image_link = image_article.find(                    'input', {'class': 'short-url-field'})[value]                # Get Tags                tag_links = image_article.find(                    'div', {'class': 'tags'}).find_all('a')                tag_names = [tag_link.string[1:] for tag_link in tag_links]                thumbnail_url = image_article.find(                    'div', {'class': 'photo-data'}).find('img')['src']                print(storing image in db)                store_image(                    image_source_link, image_link, thumbnail_url, 'FC', tag_names)                self.crawler_db.images_scraped += 1                self.crawler_db.save()            current_page += 1            self.crawler_db.current_page += 1            self.crawler_db.save()class StocksnapCrawler:    def __init__(self, crawler_db):        self.current_page = crawler_db.current_page        self.crawler_db = crawler_db    def crawl(self):        current_page = self.current_page        print(Starting crawl on page  + str(current_page))        while True:            print(crawling page  + str(current_page))            page_response = requests.get(                'https://stocksnap.io/view-photos/sort/date/desc/page-{}'.format(current_page))            page_soup = BeautifulSoup(page_response.text)            image_links = ['https://stocksnap.io' + link['href']                           for link in page_soup.find_all('a', {'class': 'photo-link'})]            for image_link in image_links:                print(scraping image at  + image_link)                response = requests.get(image_link)                image_page_soup = BeautifulSoup(response.text)                print('getting image source link')                image_source_link = image_page_soup.find(                    'img', {'class': 'img-photo'})['src']                # Get Tags                print('getting tags')                table = image_page_soup.find('table', {'class': 'img-details'})                if table:                    tag_links = table.find_all('a')                    tag_names = [tag_link.string for tag_link in tag_links if tag_link.string is not None]                thumbnail_url = image_source_link                print(storing image in db)                store_image(                    image_source_link, image_link, thumbnail_url, 'SS', tag_names)                self.crawler_db.images_scraped += 1                self.crawler_db.save()            current_page += 1            self.crawler_db.current_page += 1            self.crawler_db.save()basically each class has the same two properties and a crawl method.the crawl method follows a pretty standard structure with some variations depending on the html layout for the website it's scraping.pseudocode:current_page - page number that it is currently scrapingimage_links - a list of links each images unique page on the websiteimage_source_link - a direct url to the highest quality version of the imagethumbnail_url - a url to a smaller version of the image to be downloaded and turned into a thumbnailtags - a list of strings, each string is a tag associated with the imageorigin - a two letter uppercase string to represent the origin website of the image, e.g: SS for the website stocksnapwhile True:    response = get_page(current_page)    page_soup = BeautifulSoup(response)    image_links = get_image_links(page_soup)    for link in image_links:        response = response = requests.get(link)        page_soup = BeautifulSoup(response)        image_source_link = get_image_source_link(page_soup)        thumbnail_url = get_thumbnail_url(page_soup)        tags = get_tags(page_soup)        store_image(image_source_link, image_link, thumbnail_url, origin , tags)current_page+=1Each website would have to implement these methods differentlyAny ideas?
Web crawlers for three image sites
python;object oriented;python 3.x;web scraping
Your first problem is that the crawl function contains almost all the code. One large function is harder to reuse, read and make changes to. If you break it up things will get a lot easier.Think about each function as a task. You want them to do one thing each. For example, you could get all the image links as one function:def get_links(self, page):    page_response = requests.get(        'http://magdeleine.co/license/cc0/page/{}/'.format(current_page))    page_soup = BeautifulSoup(page_response.text)    return [link[href] for link in                page_soup.find_all('a', {'class': 'photo-link'})]But notice, you have almost the same process for all three classes shown here. The difference is the URL base that you're requesting from and the 'class' attribute. But those should both be attributes of the class. Then you could rewrite the function like this:def get_links(self, page):    page_response = requests.get(self.base_url.format(current_page))    page_soup = BeautifulSoup(page_response.text)    return [link[href] for link in                page_soup.find_all('a', {'class': self.image_class})]You understand the program and can likely form better names, but now this could be the same function in all three cases.Similarly you could turn other parts into functions too:def scrape_image(self, image_link, image_class, html_tag):    print(scraping image at  + image_link)    response = requests.get(image_link)    image_page_soup = BeautifulSoup(response.text)    print('getting image source link')    image_source_link = image_page_soup.find(        'a', {'class': image_class})[html_tag]    return image_page_soupYou can then call this with individual Crawler's attributes, like this:def crawl(self):    ...    self.scrape_image(image_link, self.scrape_image_class,                      self.scrape_html_tag)Your get_tags function is trickier, as there are entirely difference commands in different cases. But in this case you could overwrite the function instead. Did you learn about inheritance when you did OOP before? Inheritance is basically when one class takes the attributes of another and then adds to them. So in this case, perhaps you have a Crawler class that has the common attributes as well as common similar functions like scrape_image, but then you add on new features for that class, here's a quick template of how the Crawler might look:class Crawler():    def __init__():    def crawl():    def scrape_image():And now you make the MagdeleineCrawler. To inherit from Crawler just put it in brackets with the class definition.class MagdeleineCrawler(Crawler):    def __init__():        Crawler.__init__(self) # Pass parameters to Crawler in here    def get_tags():This way you can do a mix of common similar functions that get passed attributes as well as defining individual functions for each class.