<commit_before>from webob.exc import HTTPBadRequest
import webob.dec


FORWARDED_TOKENS = set(['by', 'for', 'host', 'proto'])


def handler_factory(handler):
    """Factory for handlers that set Host using RFC 7239 Forwarded header.

    :param handler: the handler that you want to wrap with Forwarded
      support
    :returns: the original ``handler`` wrapped with forwarded support.

    You can use this handler factory if you have an application that
    sits behind a trusted proxy that you know sets the RFC 7239
    Forwarded header. The Forwarded handler adjusts the Host header
    according to the information in this header.

    This makes the properties and methods on request that use this
    host information use the information the Forwarded header.

    This affects the properties and methods on :class:`BaseRequest`
    that use host information, such as :attr:`BaseRequest.host`,
    :attr:`BaseRequest.host_port`, :attr:`BaseRequest.host_url`,
    :attr:`BaseRequest.application_url`, :attr:`BaseRequest.path_url`,
    :attr:`BaseRequest.url`, :meth:`BaseRequest.relative_url`, and
    :attr:`BaseRequest.domain`.

    **Warning**: Use this handler factory *only* when you know your
    application lives behind a trusted proxy that sets the Forwarded
    header. The Forwarded header can be used by attackers to affect
    the URLs generated by your application and make them point to
    another host altogether.

    """
    def wrapped_handler(request):
        forwarded_header = request.headers.get('forwarded')
        if forwarded_header is None:
            return handler(request)
        try:
            forwarded = parse(forwarded_header)
        except ForwardedError:
            return HTTPBadRequest('Illegal Forwarded header')
        if not forwarded:
            return handler(request)
        item = forwarded[-1]
        forwarded_host = item.get('host')
        if forwarded_host is not None:
            request.host = forwarded_host
        forwarded_proto = item.get('proto')
        if forwarded_proto is not None:
            request.scheme = forwarded_proto
        return handler(request)
    return wrapped_handler


class Middleware(object):
    def __init__(self, app):
        def handler(request):
            return request.get_response(app)
        self.wrapped_handler = handler_factory(handler)

    @webob.dec.wsgify
    def __call__(self, request):
        return self.wrapped_handler(request)


class ForwardedError(ValueError):
    pass


def parse(header):
    """Parse the Forwarded header according to RFC 7239.

    Results in a list of dictionaries with the values in the header.
    """
    result = []
    for element in header.split(','):
        item = {}
        element = element.strip()
        if not element:
          continue
        for pair in element.split(';'):
            token, equals, value = pair.partition('=')
            if equals != '=':
                raise ForwardedError("Not a pair: %r" % pair)
            token = token.lower()
            # not a valid token, also deals with whitespace
            if token not in FORWARDED_TOKENS:
                raise ForwardedError("Unknown token: %r" % token)
            if token in item:
                raise ForwardedError("Duplicate token: %r" % token)
            if len(value.strip()) != len(value):
                raise ForwardedError(
                    "Value may not start or end with whitespace: %r" % value)
            # deal with quoted strings
            if value.startswith('"') and value.endswith('"'):
                value = value[1:-1]
            if value.startswith('"') or value.endswith('"'):
                raise ForwardedError(
                    "Value has unbalancd quotes: %r" % value)
            item[token] = value
        result.append(item)
    return result

<commit_msg>Get rid of PEP8 warnings<commit_after>from webob.exc import HTTPBadRequest
import webob.dec


FORWARDED_TOKENS = set(['by', 'for', 'host', 'proto'])


def handler_factory(handler):
    """Factory for handlers that set Host using RFC 7239 Forwarded header.

    :param handler: the handler that you want to wrap with Forwarded
      support
    :returns: the original ``handler`` wrapped with forwarded support.

    You can use this handler factory if you have an application that
    sits behind a trusted proxy that you know sets the RFC 7239
    Forwarded header. The Forwarded handler adjusts the Host header
    according to the information in this header.

    This makes the properties and methods on request that use this
    host information use the information the Forwarded header.

    This affects the properties and methods on :class:`BaseRequest`
    that use host information, such as :attr:`BaseRequest.host`,
    :attr:`BaseRequest.host_port`, :attr:`BaseRequest.host_url`,
    :attr:`BaseRequest.application_url`, :attr:`BaseRequest.path_url`,
    :attr:`BaseRequest.url`, :meth:`BaseRequest.relative_url`, and
    :attr:`BaseRequest.domain`.

    **Warning**: Use this handler factory *only* when you know your
    application lives behind a trusted proxy that sets the Forwarded
    header. The Forwarded header can be used by attackers to affect
    the URLs generated by your application and make them point to
    another host altogether.

    """
    def wrapped_handler(request):
        forwarded_header = request.headers.get('forwarded')
        if forwarded_header is None:
            return handler(request)
        try:
            forwarded = parse(forwarded_header)
        except ForwardedError:
            return HTTPBadRequest('Illegal Forwarded header')
        if not forwarded:
            return handler(request)
        item = forwarded[-1]
        forwarded_host = item.get('host')
        if forwarded_host is not None:
            request.host = forwarded_host
        forwarded_proto = item.get('proto')
        if forwarded_proto is not None:
            request.scheme = forwarded_proto
        return handler(request)
    return wrapped_handler


class Middleware(object):
    def __init__(self, app):
        def handler(request):
            return request.get_response(app)
        self.wrapped_handler = handler_factory(handler)

    @webob.dec.wsgify
    def __call__(self, request):
        return self.wrapped_handler(request)


class ForwardedError(ValueError):
    pass


def parse(header):
    """Parse the Forwarded header according to RFC 7239.

    Results in a list of dictionaries with the values in the header.
    """
    result = []
    for element in header.split(','):
        item = {}
        element = element.strip()
        if not element:
            continue
        for pair in element.split(';'):
            token, equals, value = pair.partition('=')
            if equals != '=':
                raise ForwardedError("Not a pair: %r" % pair)
            token = token.lower()
            # not a valid token, also deals with whitespace
            if token not in FORWARDED_TOKENS:
                raise ForwardedError("Unknown token: %r" % token)
            if token in item:
                raise ForwardedError("Duplicate token: %r" % token)
            if len(value.strip()) != len(value):
                raise ForwardedError(
                    "Value may not start or end with whitespace: %r" % value)
            # deal with quoted strings
            if value.startswith('"') and value.endswith('"'):
                value = value[1:-1]
            if value.startswith('"') or value.endswith('"'):
                raise ForwardedError(
                    "Value has unbalancd quotes: %r" % value)
            item[token] = value
        result.append(item)
    return result

{'fraction_non_alphanumeric': 0.08441222164681512, 'fraction_numerical': 0.004013464526152252, 'mean_word_length': 2.4860108303249095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2238362', 'n_tokens_mistral': 2082, 'n_tokens_neox': 1955, 'n_words': 724}