<commit_before>from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.meta.base import MetaBase

class Meta(MetaBase):
    name = 'Questionable Content'
    language = 'en'
    url = 'http://questionablecontent.net/'
    start_date = '2003-08-01'
    rights = 'Jeph Jacques'

class Crawler(CrawlerBase):
    history_capable_date = '2003-08-01'
    schedule = 'Mo,Tu,We,Th,Fr'
    time_zone = -6

    def crawl(self, pub_date):
        feed = self.parse_feed('http://www.questionablecontent.net/QCRSS.xml')
        for entry in feed.for_date(pub_date):
            url = entry.summary.src('img')
            if url == None:
                continue
            title = entry.title

            # Construct the text as the "stuff under the image" on the page / feed.
            text = "\n\n".join( [x.strip() for x in entry.html(entry.description).text( 'p', allowmultiple=True ) if len(x.strip()) > 0] )

            return CrawlerImage(url, title=title, text=text)<commit_msg>Fixed a bug where no image text crashes the crawl<commit_after>from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.meta.base import MetaBase

class Meta(MetaBase):
    name = 'Questionable Content'
    language = 'en'
    url = 'http://questionablecontent.net/'
    start_date = '2003-08-01'
    rights = 'Jeph Jacques'

class Crawler(CrawlerBase):
    history_capable_date = '2003-08-01'
    schedule = 'Mo,Tu,We,Th,Fr'
    time_zone = -6

    def crawl(self, pub_date):
        feed = self.parse_feed('http://www.questionablecontent.net/QCRSS.xml')
        for entry in feed.for_date(pub_date):
            url = entry.summary.src('img')
            if url == None:
                continue
            title = entry.title

            # Construct the text as the "stuff under the image" on the page / feed.
            text = "\n\n".join( [x.strip() for x in entry.html(entry.description).text( 'p', default=[], allowmultiple=True ) if len(x.strip()) > 0] )

            return CrawlerImage(url, title=title, text=text)
{'fraction_non_alphanumeric': 0.1232876712328767, 'fraction_numerical': 0.01761252446183953, 'mean_word_length': 2.6714542190305206, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4134879', 'n_tokens_mistral': 677, 'n_tokens_neox': 631, 'n_words': 163}