<commit_before>from comics.crawler.base import BaseComicCrawler
from comics.crawler.meta import BaseComicMeta
from comics.crawler.utils.lxmlparser import LxmlParser

class ComicMeta(BaseComicMeta):
    name = 'The Unspeakable Vault (of Doom)'
    language = 'en'
    url = 'http://www.macguff.fr/goomi/unspeakable/'
    history_capable_days = 180
    time_zone = 1
    rights = 'Francois Launet'

class ComicCrawler(BaseComicCrawler):
    def _get_url(self):
        # FIXME: The uvod feed often contains dates which feedparser fails
        # to parse, like '19 Sept 2008 00:00:00 -0800'

        self.parse_feed('http://www.macguff.fr/goomi/unspeakable/rss.xml')

        for entry in self.feed.entries:
            if (entry.updated_parsed is not None and
                self.timestamp_to_date(entry.updated_parsed) == self.pub_date
                and entry.title.startswith('Strip #')):
                self.title = entry.summary
                self.web_url = entry.link
                break

        if self.web_url is None:
            return

        page = LxmlParser(self.web_url)
        self.url = page.src('img[src*="WEBIMAGES/CARTOON"]')
        self.title = page.text('font[color="#40d265"]', default="")<commit_msg>Update uvod crawler to use their new better feed<commit_after>from comics.crawler.base import BaseComicCrawler
from comics.crawler.meta import BaseComicMeta
from comics.crawler.utils.lxmlparser import LxmlParser

class ComicMeta(BaseComicMeta):
    name = 'The Unspeakable Vault (of Doom)'
    language = 'en'
    url = 'http://www.macguff.fr/goomi/unspeakable/'
    history_capable_days = 180
    time_zone = 1
    rights = 'Francois Launet'

class ComicCrawler(BaseComicCrawler):
    def _get_url(self):
        # FIXME: The uvod feed often contains dates which feedparser fails
        # to parse, like '19 Sept 2008 00:00:00 -0800'

        self.parse_feed('http://www.macguff.fr/goomi/unspeakable/rss.xml')

        for entry in self.feed.entries:
            if (entry.updated_parsed is not None and
                self.timestamp_to_date(entry.updated_parsed) == self.pub_date
                and entry.title.startswith('Strip #')):
                self.title = entry.summary
                pieces = entry.content[0].value.split('"')
                for i, piece in enumerate(pieces):
                    if piece.count('src='):
                        self.url = pieces[i + 1]
                        return
{'fraction_non_alphanumeric': 0.10081632653061225, 'fraction_numerical': 0.019183673469387756, 'mean_word_length': 2.442415730337079, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3373790', 'n_tokens_mistral': 821, 'n_tokens_neox': 758, 'n_words': 175}