from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase


class ComicData(ComicDataBase):
    name = 'GU Comics'
    language = 'en'
    url = 'http://www.gucomics.com/'
    start_date = '2000-07-10'
    rights = 'Woody Hearn'


class Crawler(CrawlerBase):
    history_capable_date = '2000-07-10'
    schedule = 'Mo,We,Fr'
    time_zone = 'US/Eastern'

    def crawl(self, pub_date):
        page_url = 'http://www.gucomics.com/' + pub_date.strftime('%Y%m%d')
        page = self.parse_page(page_url)

        title = page.text('b', allow_multiple=True)[0]
        title = title.replace('"', '')
        title = title.strip()

        text = page.text('font[class="main"]')
        #  If there is a --- the text after is not about the comic
        text = text[:text.find('---')]
        text = text.strip()

        url = 'http://www.gucomics.com/comics/' + pub_date.strftime(
            '%Y/gu_%Y%m%d')+'.jpg'
        return CrawlerImage(url, title, text)

import re

from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase


class ComicData(ComicDataBase):
    name = 'GU Comics'
    language = 'en'
    url = 'http://www.gucomics.com/'
    start_date = '2000-07-10'
    rights = 'Woody Hearn'


class Crawler(CrawlerBase):
    history_capable_date = '2000-07-10'
    schedule = 'Mo,We,Fr'
    time_zone = 'US/Eastern'

    def crawl(self, pub_date):
        page_url = 'http://www.gucomics.com/%s' % pub_date.strftime('%Y%m%d')
        page = self.parse_page(page_url)

        title = page.text('b', allow_multiple=True)[0]
        title = title.replace('"', '')
        title = title.strip()

        text = page.text('.main')

        #  If there is a "---", the text after is not about the comic
        text = text[:text.find('---')]
        # If there is a "[ ", the text after is not part of the text
        text = text[:text.find('[ ')]
        text = text.strip()
        # Reduce any amount of newlines down to two newlines
        text = text.replace('\r', '')
        text = re.sub('\s*\n\n\s*', '\n\n', text)

        url = page.src('img[alt^="Comic for"]')
        return CrawlerImage(url, title, text)

Improve 'gucomics' text extraction