<commit_before>from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase


class ComicData(ComicDataBase):
    name = "<NAME>"
    language = "en"
    url = "http://www.awkwardzombie.com/"
    start_date = "2006-09-20"
    rights = "<NAME>"


class Crawler(CrawlerBase):
    history_capable_days = 8
    schedule = "Mo"
    time_zone = "US/Eastern"

    # Without User-Agent set, the server returns 403 Forbidden
    headers = {"User-Agent": "Mozilla/4.0"}

    def crawl(self, pub_date):
        page = self.parse_page("https://www.awkwardzombie.com/")

        page_epoch = page.src("#cc-comic")
        page_epoch = page_epoch.rsplit("/", 1)[-1]
        page_epoch = page_epoch.split("-", 1)[0]
        try:
            page_epoch = int(page_epoch)
        except ValueError:
            return

        pub_date_start = self.date_to_epoch(pub_date)
        pub_date_end = pub_date_start + 24 * 60 * 60
        if not (pub_date_start < page_epoch < pub_date_end):
            return

        result = [
            CrawlerImage(url)
            for url in page.src("#cc-comic", allow_multiple=True)
        ]
        if result:
            result[0].title = page.title("#cc-comic").strip()
        return result<commit_msg>Rewrite crawler for "Awkward Zombie" with history<commit_after>from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase


class ComicData(ComicDataBase):
    name = "<NAME>"
    language = "en"
    url = "http://www.awkwardzombie.com/"
    start_date = "2006-09-20"
    rights = "<NAME>"


class Crawler(CrawlerBase):
    history_capable_date = "2006-09-19"
    schedule = "Mo"
    time_zone = "US/Eastern"
    archive_page = None

    def crawl(self, pub_date):
        if not self.archive_page:
            page_url = "https://www.awkwardzombie.com/awkward-zombie/archive/"
            self.archive_page = self.parse_page(page_url)

        release = self.archive_page.root.xpath(
            "//div[(@class='archive-date') and contains(.,'%s')]/.."
            % pub_date.strftime("%m-%d-%y")
        )
        if not release:
            return
        release = release[0]
        title = release.xpath("div[@class='archive-title']/a")
        title = title[0]
        game = release.xpath("div[@class='archive-game']/a")
        game = game[0].text

        link = title.get("href")
        title = title.text
        release_page = self.parse_page(link)

        img = release_page.root.xpath("//img[@id='cc-comic']")
        url = img[0].get("src")

        return CrawlerImage(url, title, game)
{'fraction_non_alphanumeric': 0.1337121212121212, 'fraction_numerical': 0.017045454545454544, 'mean_word_length': 2.4613368283093053, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6176181', 'n_tokens_mistral': 914, 'n_tokens_neox': 869, 'n_words': 165}