<commit_before>import datetime as dt
import time

from comics.aggregator.exceptions import *
from comics.aggregator.feedparser import FeedParser
from comics.aggregator.lxmlparser import LxmlParser
from comics.core.models import Release, Strip

class CrawlerResult(object):
    def __init__(self, url, title=None, text=None, headers=None):
        self.url = url
        self.title = title
        self.text = text
        self.request_headers = headers or {}

    def validate(self, comic, pub_date):
        self.comic = comic
        self.pub_date = pub_date
        self._check_strip_url()

    def set_download_settings(self, check_image_mime_type, has_rerun_releases):
        self.check_image_mime_type = check_image_mime_type
        self.has_rerun_releases = has_rerun_releases

    def _check_strip_url(self):
        if not self.url:
            raise StripURLNotFound(self.identifier)

    @property
    def identifier(self):
       return u'%s/%s' % (self.comic.slug, self.pub_date)

class CrawlerBase(object):
    ### Crawler settings
    # Date of oldest release available for crawling
    history_capable_date = None
    # Number of days a release is available for crawling
    history_capable_days = None
    # On what weekdays the comic is published (example: "Mo,We,Fr")
    schedule = None
    # In approximately what time zone (in whole hours relative to UTC, without
    # regard to DST) the comic is published
    time_zone = None
    # Whether to allow multiple releases per day
    multiple_releases_per_day = False

    ### Downloader settings
    # Whether the comic reruns old strips as new releases
    has_rerun_releases = False
    # Whether to check the mime type of the strip image when downloading
    check_image_mime_type = True

    # Feed object which is reused when crawling multiple dates
    feed = None

    def __init__(self, comic):
        self.comic = comic

    def get_strip_metadata(self, pub_date=None):
        """Get URL of strip from pub_date, or the latest strip"""

        pub_date = self._get_date_to_crawl(pub_date)
        result = self.crawl(pub_date)
        if result is not None:
            result.validate(self.comic, pub_date)
            result.set_download_settings(
                check_image_mime_type=self.check_image_mime_type,
                has_rerun_releases=self.has_rerun_releases)
            if self.feed:
                result = self._decode_feed_data(result)
            return result

    def _get_date_to_crawl(self, pub_date):
        if pub_date is None:
            pub_date = dt.date.today()

        if pub_date < self.history_capable():
            raise NotHistoryCapable(self.history_capable())

        if not self.multiple_releases_per_day:
            if Release.objects.filter(comic=self.comic,
                    pub_date=pub_date).count():
                raise StripAlreadyExists('%s/%s' % (self.comic.slug, pub_date))

        return pub_date

    def history_capable(self):
        if self.history_capable_date is not None:
            return self.history_capable_date
        elif self.history_capable_days is not None:
            return (dt.date.today() - dt.timedelta(self.history_capable_days))<commit_msg>Convert history_capable_date from string to date object before use<commit_after>import datetime as dt
import time

from comics.aggregator.exceptions import *
from comics.aggregator.feedparser import FeedParser
from comics.aggregator.lxmlparser import LxmlParser
from comics.core.models import Release, Strip

class CrawlerResult(object):
    def __init__(self, url, title=None, text=None, headers=None):
        self.url = url
        self.title = title
        self.text = text
        self.request_headers = headers or {}

    def validate(self, comic, pub_date):
        self.comic = comic
        self.pub_date = pub_date
        self._check_strip_url()

    def set_download_settings(self, check_image_mime_type, has_rerun_releases):
        self.check_image_mime_type = check_image_mime_type
        self.has_rerun_releases = has_rerun_releases

    def _check_strip_url(self):
        if not self.url:
            raise StripURLNotFound(self.identifier)

    @property
    def identifier(self):
       return u'%s/%s' % (self.comic.slug, self.pub_date)

class CrawlerBase(object):
    ### Crawler settings
    # Date of oldest release available for crawling
    history_capable_date = None
    # Number of days a release is available for crawling
    history_capable_days = None
    # On what weekdays the comic is published (example: "Mo,We,Fr")
    schedule = None
    # In approximately what time zone (in whole hours relative to UTC, without
    # regard to DST) the comic is published
    time_zone = None
    # Whether to allow multiple releases per day
    multiple_releases_per_day = False

    ### Downloader settings
    # Whether the comic reruns old strips as new releases
    has_rerun_releases = False
    # Whether to check the mime type of the strip image when downloading
    check_image_mime_type = True

    # Feed object which is reused when crawling multiple dates
    feed = None

    def __init__(self, comic):
        self.comic = comic

    def get_strip_metadata(self, pub_date=None):
        """Get URL of strip from pub_date, or the latest strip"""

        pub_date = self._get_date_to_crawl(pub_date)
        result = self.crawl(pub_date)
        if result is not None:
            result.validate(self.comic, pub_date)
            result.set_download_settings(
                check_image_mime_type=self.check_image_mime_type,
                has_rerun_releases=self.has_rerun_releases)
            if self.feed:
                result = self._decode_feed_data(result)
            return result

    def _get_date_to_crawl(self, pub_date):
        if pub_date is None:
            pub_date = dt.date.today()

        if pub_date < self.history_capable():
            raise NotHistoryCapable(self.history_capable())

        if not self.multiple_releases_per_day:
            if Release.objects.filter(comic=self.comic,
                    pub_date=pub_date).count():
                raise StripAlreadyExists('%s/%s' % (self.comic.slug, pub_date))

        return pub_date

    def history_capable(self):
        if self.history_capable_date is not None:
            return dt.datetime.strptime(
                self.history_capable_date, '%Y-%m-%d').date()
        elif self.history_capable_days is not None:
            return (dt.date.today() - dt.timedelta(self.history_capable_days))
{'fraction_non_alphanumeric': 0.07233521945251567, 'fraction_numerical': 0.0, 'mean_word_length': 2.75, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4484197', 'n_tokens_mistral': 1993, 'n_tokens_neox': 1867, 'n_words': 517}