This was largely an exercise in making my code more Pythonic, especially in catching errors and doing things the right way. I opted to make the PageNotFound exception part of the class so that users could simply from ratebeer import RateBeer and not have to worry about anything else.If you prefer, the code is on Github.from bs4 import BeautifulSoupimport requestsimport reimport exceptionsclass RateBeer():        Makes getting information about beers and breweries from RateBeer.com as easy as:    >>> summit_epa = RateBeer().beer(summit extra pale ale)    A utility for searching RateBeer.com, finding information about beers, breweries, and reviews.    The nature of web scraping means that this package is offered in perpetual beta.    Requires BeautifulSoup, Requests, and lxml.    See https://github.com/alilja/ratebeer for the full README.        class PageNotFound(Exception):        pass    def __init__(self):        self.BASE_URL = http://www.ratebeer.com    def _search(self, query):        # this feels bad to me        # but if it fits, i sits        payload = {BeerName: query}        r = requests.post(self.BASE_URL+/findbeer.asp, data = payload)        return BeautifulSoup(r.text, lxml)    def _parse(self, soup):        s_results = soup.find_all('table',{'class':'results'})        output = {breweries:[],beers:[]}        beer_location = 0        # find brewery information        if any(brewers in s for s in soup.find_all(h1)):            s_breweries = s_results[0].find_all('tr')            beer_location = 1            for row in s_breweries:                location = row.find('td',{'align':'right'})                output['breweries'].append({                    name:row.a.contents,                    url:row.a.get('href'),                    id:re.search(/(?P<id>\d*)/,row.a.get('href')).group('id'),                    location:location.text.strip(),                })        # find beer information        if any(beers in s for s in soup.find_all(h1)) or not soup.find_all(text=0 beers):            s_beer_trs = iter(s_results[beer_location].find_all('tr'))            next(s_beer_trs)            for row in s_beer_trs:                 link = row.find('td','results').a                align_right = row.find_all(td,{'align':'right'})                output['beers'].append({                        name:link.text,                        url:link.get('href'),                        id:re.search(/(?P<id>\d*)/,link.get('href')).group('id'),                        rating:align_right[-2].text.strip(),                        num_ratings:align_right[-1].text,                    })        return output    def search(self, query):        return self._parse(self._search(query))    def beer(self, url):        r = requests.get(self.BASE_URL+url, allow_redirects=True)        soup = BeautifulSoup(r.text,lxml)        output = {}        # check for 404s        try:            s_contents_rows = soup.find('div',id='container').find('table').find_all('tr')        except AttributeError:            raise RateBeer.PageNotFound(url)        # ratebeer pages don't actually 404, they just send you to this weird beer reference         # page but the url doesn't actually change, it just seems like it's all getting done        # server side -- so we have to look for the contents h1 to see if we're looking at the        # beer reference or not        if beer reference in s_contents_rows[0].find_all('td')[1].h1.contents:            raise RateBeer.PageNotFound(url)        info = s_contents_rows[1].tr.find_all('td')        additional_info = s_contents_rows[1].find_all('td')[1].div.small        abbr = additional_info.find_all([abbr,a])        big = additional_info.find_all(big)        if additional_info.find(text=re.compile(SEASONAL)):            del big[2]        for location, label in enumerate(abbr):            key = None            if RATINGS in label.text:  key = num_ratings            if CALORIES in label.text: key = calories            if ABV in label.text:      key = abv            if SEASONAL in label.text: key = season            if IBU in label.text:      key = ibu            if key is not None:                output[key] = big[location].text        output.update({'name':s_contents_rows[0].find_all('td')[1].h1.text,            'overall_rating':info[0].find_all('span', attrs={'itemprop':'average'})[0].text,            'style_rating':info[0].find_all('div')[2].div.span.text,            'brewery': info[1].a.text,            'brewery_url':info[1].a.get('href'),            'style':info[1].div.find_all('a')[1].text,        })        return output    def reviews(self, url, pages=1,start_page=1,review_order=most recent):        assert pages > 0, ``pages`` must be greater than 0        assert start_page > 0, ``start_page`` must be greater than 0        review_order = review_order.lower()        url_codes = {            most recent:1,            top raters:2,            highest score:3            }        url_flag = url_codes.get(review_order)        if not url_flag: raise ValueError, Invalid ``review_order``.        output = []        for page_number in range(start_page,start_page+pages):            complete_url = {0}{1}{2}/{3}/.format(self.BASE_URL, url, url_flag, page_number)            r = requests.get(complete_url, allow_redirects=True)            soup = BeautifulSoup(r.text, lxml)            content = soup.find('div',id='container').find('table').find_all('tr')[5]            [x.extract() for x in content.find_all('table')] # strip ad section            review_tuples = zip(*[iter(content.find_all('div'))] * 4) # basically magic            for review in review_tuples:                detail_tuples = zip(*[iter(review[0].find_all([big,small]))] * 2)                details = dict([(                        label.text.lower().strip().encode(ascii,ignore),                         rating.text,                    ) for (label, rating) in detail_tuples])                details.update({'text':review[3].text})                output.append(details)        return output    def brewery(self, url, include_beers=True):        r = requests.get(self.BASE_URL+url, allow_redirects=True)        soup = BeautifulSoup(r.text, lxml)        try:            s_contents = soup.find('div',id='container').find('table').find_all('tr')[0].find_all('td')        except AttributeError:            raise RateBeer.PageNotFound(url)        output = {            'name':s_contents[8].h1.text,            'type':re.search(Type: +(?P<type>[^ ]+),s_contents[8].find_all('span','beerfoot')[1].text).group('type'),            'street':s_contents[0].find('span',attrs={'itemprop':'streetAddress'}).text,            'city':s_contents[0].find('span',attrs={'itemprop':'addressLocality'}).text,            'state':s_contents[0].find('span',attrs={'itemprop':'addressRegion'}).text,            'country':s_contents[0].find('span',attrs={'itemprop':'addressCountry'}).text,            'postal_code':s_contents[0].find('span',attrs={'itemprop':'postalCode'}).text,            }        if include_beers:            output.update({'beers':[]})            s_beer_trs = iter(s_contents[8].find('table','maintable nohover').find_all('tr'))            next(s_beer_trs)            for row in s_beer_trs:                beer = {                    'name':row.a.text,                    'url':row.a.get('href'),                    'id':re.search(/(?P<id>\d*)/,row.a.get('href')).group('id'),                    'rating':row.find_all('td')[4].text.strip(),                    'num_ratings':row.find_all('td')[6].text.strip(),                }                output['beers'].append(beer)        return output
RateBeer.com scraper
python;error handling;library;web scraping;beautifulsoup
I ran pylint on your code, which came up with 98 warnings - this is not good! I have split this down into each warning class, ordered by number of occurrences.Note that if you particularly disagree with any specific warning, you can use comments to disable them, e.g.:# pylint: disable=too-many-localsThis at least makes it clear that you've considered and dismissed the issue.bad-whitespace [30 counts]The style guide lays out the rules for whitespace, you should have:One space after commas (e.g. in tuples, lists, definitions and calls);One space after colons (e.g. in dictionaries); andNo spaces around = in calls/definitions.Not only do you not follow this, you are inconsistent. Also, lining up e.g.if RATINGS in label.text:  key = num_ratingsif CALORIES in label.text: key = calorieswith whitespace is not Pythonic (we will come back to this line later...)line-too-long [24 counts]Lines should be at most 79 characters long (72 for docstrings). One is 119 characters long!bad-continuation [13 counts]When continuing statements across multiple lines, you should eitheracceptable = ('start on the first line' and              'continue at the same indent') # close paren on same lineoralso_acceptable = (    'start on the next line' and    ' use a single level of indent') # closing paren on next line, here#   ) or herePick one of the three, and be consistent.missing-docstring [8 counts]Although you have a docstring for the class (albeit not correctly formatted), none of its methods are documented. I would put your overall explanation as a module docstring, then have an addition docstring for at least every public method. I like the Google style, but there are others.multiple-statements [6 counts]To return, as promised, to:if RATINGS in label.text:  key = num_ratingsThat line contains two statements, and should therefore be on two lines:if RATINGS in label.text:      key = num_ratingsNote that you could use a dictionary to simplify this, for example:KEYWORDS = {'RATINGS': 'num_ratings', ...}for keyword in KEYWORDS:    if keyword in label.text:        key = KEYWORDS[keyword]        breakinvalid-name [5 counts]A few methods use r as a variable name - something more meaningful might be helpful (pylint generally requires at least three characters).Also, BASE_URL should be a class attribute.trailing-whitespace [3 counts]There should be no tabs/spaces on blank lines or at the end of a line. You can generally set up IDEs to remove this automatically (or when you ask them to - e.g. Format > Strip trailing whitespace in IDLE). If you have the option, turning on display of whitespace helps you see what's going on.anomalous-backslash-in-string [3 counts]You have a few regular expressions that use backslashes (e.g. for digits \d) - these should be made raw strings r'...'.unused-import [1 count]You never use exceptionstoo-many-locals [1 count]There are 20 local variables in RateBeer.beer, suggesting it should be split into more, smaller methods (pylint sets the limit at 15).old-style-class [1 count]RateBeer should inherit from object to make it a new-style class:class RateBeer(object):old-raise-syntax [1 count]You mostly raise correctly, but:if not url_flag: raise ValueError, Invalid ``review_order``.Again, one statement per line, and use the new syntax:if not url_flag:    raise ValueError(Invalid ``review_order``.)no-self-use [1 count]RateBeer._parse doesn't actually use any instance attributes, and could therefore be a standalone function or @staticmethod.expression-not-assigned [1 count]This line:[x.extract() for x in content.find_all('table')] creates a list that never gets assigned. If you're just doing this for the side-effects, make that clear with e.g.:_ = [x.extract() for x in content.find_all('table')] Additionally, the import statements should be:import re # standard library firstfrom bs4 import BeautifulSoup #then third-partyimport requests