<commit_before>from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.meta.base import MetaBase
import re

class Meta(MetaBase):
    name = 'Yehuda Moon'
    language = 'en'
    url = 'http://www.yehudamoon.com/'
    start_date = '2008-01-22'
    rights = '(c) 2010 <NAME>'

class Crawler(CrawlerBase):
    history_capable_date = '2008-01-22'
    schedule = 'Mo,Tu,We,Th,Fr,Sa,Su'
    time_zone = -5
    has_rerun_releases = False

    def crawl(self, pub_date):
        page_url = 'http://www.yehudamoon.com/index.php?date=%s' % pub_date.strftime( "%Y-%m-%d" )
        page = self.parse_page( page_url )

        # It'll forward you to the most current day if it doesn't have
        # pub_date. Check *explicitly* to make sure this day exists and bug out
        # if not
        current_day = page.value( 'select[id=ss_select] option[value*=%s]' % pub_date.strftime( "%Y-%m-%d" ) )
        if current_day == None:
            return CrawlerImage( None )

        url = page.src( 'div[id="ss_img_div"] img' )
        title_full = page.text( 'option[value*="%s"]' % pub_date.strftime( "%Y-%m-%d" ) )
        title = re.sub( '^.*- *', '', title_full )
        return CrawlerImage(url, title=title)<commit_msg>Commited part 2 of the Yehuda Moon fix<commit_after>from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.meta.base import MetaBase
import re

class Meta(MetaBase):
    name = 'Yehuda Moon'
    language = 'en'
    url = 'http://www.yehudamoon.com/'
    start_date = '2008-01-22'
    rights = '(c) 2010 <NAME>'

class Crawler(CrawlerBase):
    history_capable_date = '2008-01-22'
    schedule = 'Mo,Tu,We,Th,Fr,Sa,Su'
    time_zone = -5
    has_rerun_releases = False

    def crawl(self, pub_date):
        page_url = 'http://www.yehudamoon.com/index.php?date=%s' % pub_date.strftime( "%Y-%m-%d" )
        page = self.parse_page( page_url )

        # It'll forward you to the most current day if it doesn't have
        # pub_date. Check *explicitly* to make sure this day exists and bug out
        # if not
        current_day = page.value( 'select[id=ss_select] option[value*=%s]' % pub_date.strftime( "%Y-%m-%d" ) )
        if current_day == None:
            return CrawlerImage( None )

        url = page.src( 'div[id="ss_img_div"] img' )
        # If we can't figure the title out, just don't store it
        try:
            title_full = page.text( 'option[value*="%s"]' % pub_date.strftime( "%Y-%m-%d" ) )
            title = re.sub( '^.*- *', '', title_full )
        except TypeError, e:
            title = None
        return CrawlerImage(url, title=title)
{'fraction_non_alphanumeric': 0.14176829268292682, 'fraction_numerical': 0.01638719512195122, 'mean_word_length': 2.755364806866953, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3279316', 'n_tokens_mistral': 974, 'n_tokens_neox': 907, 'n_words': 221}