<commit_before>import datetime as dt
import time
import urllib2

from django.conf import settings

from comics.aggregator.exceptions import (CrawlerHTTPError, ImageURLNotFound,
    NotHistoryCapable, ReleaseAlreadyExists)
from comics.aggregator.feedparser import FeedParser
from comics.aggregator.lxmlparser import LxmlParser
from comics.core.models import Release, Strip

class CrawlerResult(object):
    def __init__(self, url, title=None, text=None, headers=None):
        self.url = url
        self.title = title
        self.text = text
        self.request_headers = headers or {}

    def validate(self, comic, pub_date):
        self.comic = comic
        self.pub_date = pub_date
        self._check_strip_url()

    def set_download_settings(self, check_image_mime_type, has_rerun_releases):
        self.check_image_mime_type = check_image_mime_type
        self.has_rerun_releases = has_rerun_releases

    def _check_strip_url(self):
        if not self.url:
            raise ImageURLNotFound(self.identifier)

    @property
    def identifier(self):
       return u'%s/%s' % (self.comic.slug, self.pub_date)

class CrawlerBase(object):
    ### Crawler settings<commit_msg>Decode strip title/text as UTF-8/ISO-8859-1 to fix 'cant adapt' errors from psycopg2<commit_after>import datetime as dt
import time
import urllib2

from django.conf import settings

from comics.aggregator.exceptions import (CrawlerHTTPError, ImageURLNotFound,
    NotHistoryCapable, ReleaseAlreadyExists)
from comics.aggregator.feedparser import FeedParser
from comics.aggregator.lxmlparser import LxmlParser
from comics.core.models import Release, Strip

class CrawlerResult(object):
    def __init__(self, url, title=None, text=None, headers=None):
        self.url = url
        self.title = title
        self.text = text
        self.request_headers = headers or {}

    def validate(self, comic, pub_date):
        self.comic = comic
        self.pub_date = pub_date
        self._check_strip_url()

    def set_download_settings(self, check_image_mime_type, has_rerun_releases):
        self.check_image_mime_type = check_image_mime_type
        self.has_rerun_releases = has_rerun_releases

    def _check_strip_url(self):
        if not self.url:
            raise ImageURLNotFound(self.identifier)

    @property
    def title(self):
        return self._title

    @title.setter
    def title(self, value):
        try:
            self._title = value and value.decode('utf-8')
        except UnicodeDecodeError:
            self._title = value and value.decode('iso-8859-1')

    @property
    def text(self):
        return self._text

    @text.setter
    def text(self, value):
        try:
            self._text = value and value.decode('utf-8')
        except UnicodeDecodeError:
            self._text = value and value.decode('iso-8859-1')

    @property
    def identifier(self):
       return u'%s/%s' % (self.comic.slug, self.pub_date)

class CrawlerBase(object):
    ### Crawler settings
{'fraction_non_alphanumeric': 0.08369456485495165, 'fraction_numerical': 0.007002334111370457, 'mean_word_length': 3.1379310344827585, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4011506', 'n_tokens_mistral': 942, 'n_tokens_neox': 885, 'n_words': 218}