common middleware for taking care of some basic operations forbid access to useragents in settings disalloweduseragents url rewriting based on the appendslash and prependwww settings append missing slashes andor prepends missing www s if appendslash is set and the initial url doesn t end with a slash and it is not found in urlpatterns form a new url by appending a slash at the end if this new url is found in urlpatterns return an http redirect to this new url otherwise process the initial url as usual this behavior can be customized by subclassing commonmiddleware and overriding the responseredirectclass attribute check for denied useragents and rewrite the url based on settings appendslash and settings prependwww check for denied useragents check for a redirect based on settings prependwww check if we also need to append a slash so we can do it all with a single redirect this check may be somewhat expensive so we only do it if we already know we re sending a redirect or in processresponse if we get a 404 return true if settings appendslash is true and appending a slash to the request path turns an invalid path into a valid one return the full path of the request with a trailing slash appended raise a runtimeerror if settings debug is true and request method is post put or patch prevent construction of scheme relative urls when the status code of the response is 404 it may redirect to a path with an appended slash if shouldredirectwithslash returns true if the given url is not found then check if we should redirect to a path with a slash appended add the contentlength header to nonstreaming responses if not already set send broken link emails for relevant 404 not found responses if response statuscode 404 and not settings debug domain request gethost path request getfullpath referer request meta gethttpreferer if not self isignorablerequestrequest path domain referer ua request meta gethttpuseragent none ip request meta getremoteaddr none mailmanagers broken slink on s internal if self isinternalrequestdomain referer else domain referrer snrequested url snuser agent sn ip address sn referer path ua ip failsilentlytrue return response def isinternalrequestself domain referer different subdomains are treated as different domains return boolre matchhttps s re escapedomain referer def isignorablerequestself request uri domain referer the referer is empty if not referer return true appendslash is enabled and the referer is equal to the current url without a trailing slash indicating an internal redirect if settings appendslash and uri endswith and referer uri 1 return true a in referer is identified as a search engine source if not self isinternalrequestdomain referer and in referer return true the referer is equal to the current url ignoring the scheme assumed to be a poorly implemented bot parsedreferer urlparsereferer if parsedreferer netloc in domain and parsedreferer path uri return true return anypattern searchuri for pattern in settings ignorable404urls common middleware for taking care of some basic operations forbid access to user agents in settings disallowed_user_agents url rewriting based on the append_slash and prepend_www settings append missing slashes and or prepends missing www s if append_slash is set and the initial url doesn t end with a slash and it is not found in urlpatterns form a new url by appending a slash at the end if this new url is found in urlpatterns return an http redirect to this new url otherwise process the initial url as usual this behavior can be customized by subclassing commonmiddleware and overriding the response_redirect_class attribute check for denied user agents and rewrite the url based on settings append_slash and settings prepend_www check for denied user agents check for a redirect based on settings prepend_www check if we also need to append a slash so we can do it all with a single redirect this check may be somewhat expensive so we only do it if we already know we re sending a redirect or in process_response if we get a 404 return true if settings append_slash is true and appending a slash to the request path turns an invalid path into a valid one return the full path of the request with a trailing slash appended raise a runtimeerror if settings debug is true and request method is post put or patch prevent construction of scheme relative urls when the status code of the response is 404 it may redirect to a path with an appended slash if should_redirect_with_slash returns true if the given url is not found then check if we should redirect to a path with a slash appended add the content length header to non streaming responses if not already set send broken link emails for relevant 404 not found responses return true if the referring url is the same domain as the current request different subdomains are treated as different domains return true if the given request shouldn t notify the site managers according to project settings or in situations outlined by the inline comments the referer is empty append_slash is enabled and the referer is equal to the current url without a trailing slash indicating an internal redirect a in referer is identified as a search engine source the referer is equal to the current url ignoring the scheme assumed to be a poorly implemented bot
import re
from urllib.parse import urlparse

from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.core.mail import mail_managers
from django.http import HttpResponsePermanentRedirect
from django.urls import is_valid_path
from django.utils.deprecation import MiddlewareMixin
from django.utils.http import escape_leading_slashes


class CommonMiddleware(MiddlewareMixin):
    

    response_redirect_class = HttpResponsePermanentRedirect

    def process_request(self, request):
        

        
        user_agent = request.META.get("HTTP_USER_AGENT")
        if user_agent is not None:
            for user_agent_regex in settings.DISALLOWED_USER_AGENTS:
                if user_agent_regex.search(user_agent):
                    raise PermissionDenied("Forbidden user agent")

        
        host = request.get_host()

        if settings.PREPEND_WWW and host and not host.startswith("www."):
            
            
            
            
            if self.should_redirect_with_slash(request):
                path = self.get_full_path_with_slash(request)
            else:
                path = request.get_full_path()

            return self.response_redirect_class(f"{request.scheme}://www.{host}{path}")

    def should_redirect_with_slash(self, request):
        
        if settings.APPEND_SLASH and not request.path_info.endswith("/"):
            urlconf = getattr(request, "urlconf", None)
            if not is_valid_path(request.path_info, urlconf):
                match = is_valid_path("%s/" % request.path_info, urlconf)
                if match:
                    view = match.func
                    return getattr(view, "should_append_slash", True)
        return False

    def get_full_path_with_slash(self, request):
        
        new_path = request.get_full_path(force_append_slash=True)
        
        new_path = escape_leading_slashes(new_path)
        if settings.DEBUG and request.method in ("POST", "PUT", "PATCH"):
            raise RuntimeError(
                "You called this URL via %(method)s, but the URL doesn't end "
                "in a slash and you have APPEND_SLASH set. Django can't "
                "redirect to the slash URL while maintaining %(method)s data. "
                "Change your form to point to %(url)s (note the trailing "
                "slash), or set APPEND_SLASH=False in your Django settings."
                % {
                    "method": request.method,
                    "url": request.get_host() + new_path,
                }
            )
        return new_path

    def process_response(self, request, response):
        
        
        
        if response.status_code == 404 and self.should_redirect_with_slash(request):
            return self.response_redirect_class(self.get_full_path_with_slash(request))

        
        
        if not response.streaming and not response.has_header("Content-Length"):
            response.headers["Content-Length"] = str(len(response.content))

        return response


class BrokenLinkEmailsMiddleware(MiddlewareMixin):
    def process_response(self, request, response):
        
        if response.status_code == 404 and not settings.DEBUG:
            domain = request.get_host()
            path = request.get_full_path()
            referer = request.META.get("HTTP_REFERER", "")

            if not self.is_ignorable_request(request, path, domain, referer):
                ua = request.META.get("HTTP_USER_AGENT", "<none>")
                ip = request.META.get("REMOTE_ADDR", "<none>")
                mail_managers(
                    "Broken %slink on %s"
                    % (
                        (
                            "INTERNAL "
                            if self.is_internal_request(domain, referer)
                            else ""
                        ),
                        domain,
                    ),
                    "Referrer: %s\nRequested URL: %s\nUser agent: %s\n"
                    "IP address: %s\n" % (referer, path, ua, ip),
                    fail_silently=True,
                )
        return response

    def is_internal_request(self, domain, referer):
        
        
        return bool(re.match("^https?://%s/" % re.escape(domain), referer))

    def is_ignorable_request(self, request, uri, domain, referer):
        
        
        if not referer:
            return True

        
        
        if settings.APPEND_SLASH and uri.endswith("/") and referer == uri[:-1]:
            return True

        
        if not self.is_internal_request(domain, referer) and "?" in referer:
            return True

        
        
        parsed_referer = urlparse(referer)
        if parsed_referer.netloc in ["", domain] and parsed_referer.path == uri:
            return True

        return any(pattern.search(uri) for pattern in settings.IGNORABLE_404_URLS)