<commit_before># The contents of this file are subject to the Mozilla Public License
# Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#    http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# OS2Webscanner was developed by Magenta n collaboration with OS2 the
# Danish community of open source municipalities (http://www.os2web.dk/).
#
# The code is currently governed by OS2 the Danish community of open
# source municipalities ( http://www.os2web.dk/ )
"""Middleware for the scanner."""

import datetime
import pytz
from lxml import html

import arrow

from scrapy import Request
from scrapy import signals
import logging
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.url import canonicalize_url

from email.utils import parsedate_tz, mktime_tz
from os2webscanner.models.urllastmodified_model import UrlLastModified

from django.conf import settings as django_settings


class ExclusionRuleMiddleware(object):

    """A Spider Middleware which excludes certain URLs from being followed.

    If the spider has an exclusion_rules attribute, this is used to determine
    whether to visit the URLs.
    The exclusion_rules is a list which can be either strings or compiled
    regular expressions.
    """

    def process_spider_output(self, response, result, spider):
        """Process the output of a spider."""
        for x in result:
            if isinstance(x, Request):
                if self.should_follow(x, spider):
                    yield x
                else:
                    # TODO: Log excluded URLs somewhere?
                    logging.info("Excluding %s" % x.url)
            else:
                yield x

    def should_follow(self, request, spider):
        """Return whether the URL should be followed."""
        return not spider.is_excluded(request)


class NoSubdomainOffsiteMiddleware(OffsiteMiddleware):

    """Offsite middleware which doesn't allow subdomains of allowed_domains."""

    def get_host_regex(self, spider):
        """Disallow subdomains of the allowed domains.

        Overrides OffsiteMiddleware.
        """
        return spider.get_host_regex()


class CookieCollectorMiddleware(CookiesMiddleware):

    """Collect all cookies set while scanning."""

    def process_response(self, request, response, spider):

        """Collect cookie, store on scan object."""

        # First, extract cookies as needed - call superclass.
        response = super().process_response(request, response, spider)

        # Now collect cookie
        current_scan = spider.scanner.scan_object
        domain = urlparse_cached(request).hostname

        if current_scan.do_collect_cookies:

            for cookie in response.headers.getlist('Set-Cookie'):
                current_scan.logging_cookie(domain + '|' + cookie)

        return response


class OffsiteDownloaderMiddleware(object):

    """Offsite middleware which doesn't allow subdomains of allowed_domains."""

    @classmethod
    def from_crawler(cls, crawler):
        """Connect spider open signal."""<commit_msg>* Now checks scanner type before checking properties.<commit_after># The contents of this file are subject to the Mozilla Public License
# Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#    http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# OS2Webscanner was developed by Magenta n collaboration with OS2 the
# Danish community of open source municipalities (http://www.os2web.dk/).
#
# The code is currently governed by OS2 the Danish community of open
# source municipalities ( http://www.os2web.dk/ )
"""Middleware for the scanner."""

import datetime
import pytz
from lxml import html

import arrow

from scrapy import Request
from scrapy import signals
import logging
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.url import canonicalize_url

from email.utils import parsedate_tz, mktime_tz
from os2webscanner.models.urllastmodified_model import UrlLastModified

from django.conf import settings as django_settings


class ExclusionRuleMiddleware(object):

    """A Spider Middleware which excludes certain URLs from being followed.

    If the spider has an exclusion_rules attribute, this is used to determine
    whether to visit the URLs.
    The exclusion_rules is a list which can be either strings or compiled
    regular expressions.
    """

    def process_spider_output(self, response, result, spider):
        """Process the output of a spider."""
        for x in result:
            if isinstance(x, Request):
                if self.should_follow(x, spider):
                    yield x
                else:
                    # TODO: Log excluded URLs somewhere?
                    logging.info("Excluding %s" % x.url)
            else:
                yield x

    def should_follow(self, request, spider):
        """Return whether the URL should be followed."""
        return not spider.is_excluded(request)


class NoSubdomainOffsiteMiddleware(OffsiteMiddleware):

    """Offsite middleware which doesn't allow subdomains of allowed_domains."""

    def get_host_regex(self, spider):
        """Disallow subdomains of the allowed domains.

        Overrides OffsiteMiddleware.
        """
        return spider.get_host_regex()


class CookieCollectorMiddleware(CookiesMiddleware):

    """Collect all cookies set while scanning."""

    def process_response(self, request, response, spider):

        """Collect cookie, store on scan object."""

        # First, extract cookies as needed - call superclass.
        response = super().process_response(request, response, spider)

        # Now collect cookie
        current_scan = spider.scanner.scan_object
        domain = urlparse_cached(request).hostname
        if hasattr(current_scan, 'webscan'):
            if current_scan.webscan.do_collect_cookies:

                for cookie in response.headers.getlist('Set-Cookie'):
                    current_scan.logging_cookie(domain + '|' + cookie)

        return response


class OffsiteDownloaderMiddleware(object):

    """Offsite middleware which doesn't allow subdomains of allowed_domains."""

    @classmethod
    def from_crawler(cls, crawler):
        """Connect spider open signal."""
{'fraction_non_alphanumeric': 0.07060597751576639, 'fraction_numerical': 0.0021935837674801205, 'mean_word_length': 3.311465721040189, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5145609', 'n_tokens_mistral': 1922, 'n_tokens_neox': 1808, 'n_words': 705}