<commit_before># -*- coding: utf-8 -*-


from urllib.parse import urlparse

from scrapy.spiders import Spider
from scrapy import Request
from scrapy.spidermiddlewares.httperror import HttpError
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError


class ErrbackSpider(Spider):

    def start_requests(self):
        for url in self.start_urls:
            yield Request(url,
                          callback=self.parse,
                          errback=self.errback)

    def errback(self, failure):
        if failure.check(DNSLookupError):
            host = urlparse(failure.request.url).hostname
            self.logger.error('DNSLookupError on domain[%s]',
                              host)
        elif failure.check(TimeoutError, TCPTimedOutError):
            request = failure.request
            self.logger.error('TimeoutError on url[%s]',
                              request.url)
        elif failure.check(HttpError):
            response = failure.value.response
            self.logger.error('HttpError on url[%s, code=%d]',<commit_msg>revert<commit_after># -*- coding: utf-8 -*-


from urllib.parse import urlparse

from scrapy.spiders import Spider
from scrapy import Request
from scrapy.spidermiddlewares.httperror import HttpError
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError


class ErrbackSpider(Spider):

    def start_requests(self):
        for url in self.start_urls:
            yield Request(url,
                          callback=self.parse,
                          errback=self.errback,
                          dont_filter=True)

    def errback(self, failure):
        if failure.check(DNSLookupError):
            host = urlparse(failure.request.url).hostname
            self.logger.error('DNSLookupError on domain[%s]',
                              host)
        elif failure.check(TimeoutError, TCPTimedOutError):
            request = failure.request
            self.logger.error('TimeoutError on url[%s]',
                              request.url)
        elif failure.check(HttpError):
            response = failure.value.response
            self.logger.error('HttpError on url[%s, code=%d]',
{'fraction_non_alphanumeric': 0.08680403331871986, 'fraction_numerical': 0.0008768084173608067, 'mean_word_length': 2.160664819944598, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5804297', 'n_tokens_mistral': 623, 'n_tokens_neox': 607, 'n_words': 136}