<commit_before>import httplib2

from BeautifulSoup import BeautifulSoup, SoupStrainer

from .pastesource import PasteSource


class PastebinSource(PasteSource):
    baseurl = 'http://pastebin.com'

    def new_urls(self, backend):
        http = httplib2.Http()
        status, response = http.request('http://pastebin.com/archive')
        product = SoupStrainer("td", {"class": "icon"})
        soup = BeautifulSoup(response, parseOnlyThese=product)

        for link in soup.findAll("a"):
            app = link["href"]
            if not backend.already_visited_url(self.full_url(app)):
                yield self, app

    def get_paste(self, path):
        http = httplib2.Http()
        status, response = http.request(self.full_url(path))
        try: #wrap parser to avoid malformed error
            feast = BeautifulSoup(response,
                                parseOnlyThese=SoupStrainer("textarea"))
        except: #return a blank which will 
            feast = ""

        return status, feast

    def full_url(self, path):
        return self.baseurl + path<commit_msg>made the exception handling for pastebin more specific and addded diagnostic printing to stderr<commit_after>import httplib2
import sys

from HTMLParser import HTMLParseError

from BeautifulSoup import BeautifulSoup, SoupStrainer

from .pastesource import PasteSource


class PastebinSource(PasteSource):
    baseurl = 'http://pastebin.com'

    def new_urls(self, backend):
        http = httplib2.Http()
        status, response = http.request('http://pastebin.com/archive')
        product = SoupStrainer("td", {"class": "icon"})
        soup = BeautifulSoup(response, parseOnlyThese=product)

        for link in soup.findAll("a"):
            app = link["href"]
            if not backend.already_visited_url(self.full_url(app)):
                yield self, app

    def get_paste(self, path):
        http = httplib2.Http()
        status, response = http.request(self.full_url(path))
        try: #wrap parser to avoid malformed error
            feast = BeautifulSoup(response,
                                parseOnlyThese=SoupStrainer("textarea"))
        except HTMLParseError as e: #return a blank which will 
            print >> sys.stderr, "failed on get_paste for path '%s': %s" % (
                    path, e)
            feast = ""

        return status, feast

    def full_url(self, path):
        return self.baseurl + path
{'fraction_non_alphanumeric': 0.09729064039408868, 'fraction_numerical': 0.0024630541871921183, 'mean_word_length': 2.486409155937053, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363703', 'n_tokens_mistral': 708, 'n_tokens_neox': 679, 'n_words': 184}