<commit_before># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import threading
import sys
import cgi
import socket
import errno
import time
import base64
from contextlib import contextmanager
from mapproxy.util.py import reraise
from mapproxy.compat import iteritems, PY2
from mapproxy.compat.modules import urlparse
if PY2:
    from cStringIO import StringIO
else:
    from io import StringIO

if PY2:
    from BaseHTTPServer import HTTPServer as HTTPServer_, BaseHTTPRequestHandler
else:
    from http.server import HTTPServer as HTTPServer_, BaseHTTPRequestHandler

class RequestsMissmatchError(AssertionError):
    def __init__(self, assertions):
        self.assertions = assertions

    def __str__(self):
        assertions = []
        for assertion in self.assertions:
            assertions.append(text_indent(str(assertion), '    ', ' -  '))
        return 'requests missmatch:\n' + '\n'.join(assertions)

class RequestError(str):
    pass

def text_indent(text, indent, first_indent=None):
    if first_indent is None:
        first_indent = indent

    text = first_indent + text
    return text.replace('\n', '\n' + indent)

class RequestMissmatch(object):
    def __init__(self, msg, expected, actual):
        self.msg = msg
        self.expected = expected
        self.actual = actual

    def __str__(self):
        return ('requests missmatch, expected:\n' +
            text_indent(str(self.expected), '    ') +
            '\n  got:\n' + text_indent(str(self.actual), '    '))

class HTTPServer(HTTPServer_):
    allow_reuse_address = True

    def handle_error(self, request, client_address):
        _exc_class, exc, _tb = sys.exc_info()
        if isinstance(exc, socket.error):
            if exc.errno == errno.EPIPE:
                # suppres 'Broken pipe' errors raised in timeout tests
                return
        HTTPServer_.handle_error(self, request, client_address)

class ThreadedStopableHTTPServer(threading.Thread):
    def __init__(self, address, requests_responses, unordered=False, query_comparator=None):
        threading.Thread.__init__(self, **{'group': None})
        self.requests_responses = requests_responses
        self.daemon = True
        self.sucess = False
        self.shutdown = False
        self.httpd = HTTPServer(address,mock_http_handler(requests_responses,
            unordered=unordered, query_comparator=query_comparator))
        self.httpd.timeout = 1.0
        self.assertions = self.httpd.assertions = []

    @property
    def http_port(self):
        return self.httpd.socket.getsockname()[1]

    def run(self):
        while self.requests_responses:
            if self.shutdown: break
            self.httpd.handle_request()
        if self.requests_responses:
            missing_req = [req for req, resp in self.requests_responses]
            self.assertions.append(
                RequestError('missing requests: ' + ','.join(map(str, missing_req)))
            )
        if not self.assertions:
            self.sucess = True
        # force socket close so next test can bind to same address
        self.httpd.socket.close()

class ThreadedSingleRequestHTTPServer(threading.Thread):
    def __init__(self, address, request_handler):
        threading.Thread.__init__(self, **{'group': None})
        self.daemon = True
        self.sucess = False
        self.shutdown = False
        self.httpd = HTTPServer(address, request_handler)
        self.httpd.timeout = 1.0
        self.assertions = self.httpd.assertions = []

    def run(self):
        self.httpd.handle_request()
        if not self.assertions:
            self.sucess = True
        # force socket close so next test can bind to same address
        self.httpd.socket.close()


def mock_http_handler(requests_responses, unordered=False, query_comparator=None):
    if query_comparator is None:
        query_comparator = query_eq
    class MockHTTPHandler(BaseHTTPRequestHandler):
        def do_GET(self):
            self.query_data = self.path
            return self.do_mock_request('GET')

        def do_POST(self):
            length = int(self.headers['content-length'])
            self.query_data = self.path + '?' + self.rfile.read(length).decode('utf-8')
            return self.do_mock_request('POST')

        def _matching_req_resp(self):
            if len(requests_responses) == 0:
                return None, None
            if unordered:
                for req_resp in requests_responses:
                    req, resp = req_resp
                    if query_comparator(req['path'], self.query_data):
                        requests_responses.remove(req_resp)
                        return req, resp
                return None, None
            else:
                return requests_responses.pop(0)

        def do_mock_request(self, method):
            req, resp = self._matching_req_resp()
            if not req:
                self.server.assertions.append(
                    RequestError('got unexpected request: %s' % self.query_data)
                )
                return
            if 'method' in req:
                if req['method'] != method:
                    self.server.assertions.append(
                        RequestMissmatch('unexpected method', req['method'], method)
                    )
                    self.server.shutdown = True
            if req.get('require_basic_auth', False):
                if 'Authorization' not in self.headers:
                    requests_responses.insert(0, (req, resp)) # push back
                    self.send_response(401)
                    self.send_header('WWW-Authenticate', 'Basic realm="Secure Area"')
                    self.end_headers()
                    self.wfile.write(b'no access')
                    return
            if req.get('headers'):
                for k, v in req['headers'].items():
                    if k not in self.headers:
                        self.server.assertions.append(
                            RequestMissmatch('missing header', k, self.headers)
                        )
                    elif self.headers[k] != v:
                        self.server.assertions.append(
                            RequestMissmatch('header missmatch', '%s: %s' % (k, v), self.headers)
                        )
            if not query_comparator(req['path'], self.query_data):
                self.server.assertions.append(
                    RequestMissmatch('requests differ', req['path'], self.query_data)
                )
                query_actual = set(query_to_dict(self.query_data).items())
                query_expected = set(query_to_dict(req['path']).items())
                self.server.assertions.append(
                    RequestMissmatch('requests params differ', query_actual - query_expected, query_expected - query_actual)
                )
                self.server.shutdown = True
            if 'req_assert_function' in req:
                if not req['req_assert_function'](self):
                    self.server.assertions.append(
                        RequestError('req_assert_function failed')
                    )
                    self.server.shutdown = True
            if 'duration' in resp:
                time.sleep(float(resp['duration']))
            self.start_response(resp)
            if resp.get('body_file'):
                with open(resp['body_file'], 'rb') as f:<commit_msg>fix expected/actual output in MockServ errors<commit_after># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import threading
import sys
import cgi
import socket
import errno
import time
import base64
from contextlib import contextmanager
from mapproxy.util.py import reraise
from mapproxy.compat import iteritems, PY2
from mapproxy.compat.modules import urlparse
if PY2:
    from cStringIO import StringIO
else:
    from io import StringIO

if PY2:
    from BaseHTTPServer import HTTPServer as HTTPServer_, BaseHTTPRequestHandler
else:
    from http.server import HTTPServer as HTTPServer_, BaseHTTPRequestHandler

class RequestsMissmatchError(AssertionError):
    def __init__(self, assertions):
        self.assertions = assertions

    def __str__(self):
        assertions = []
        for assertion in self.assertions:
            assertions.append(text_indent(str(assertion), '    ', ' -  '))
        return 'requests missmatch:\n' + '\n'.join(assertions)

class RequestError(str):
    pass

def text_indent(text, indent, first_indent=None):
    if first_indent is None:
        first_indent = indent

    text = first_indent + text
    return text.replace('\n', '\n' + indent)

class RequestMissmatch(object):
    def __init__(self, msg, expected, actual):
        self.msg = msg
        self.expected = expected
        self.actual = actual

    def __str__(self):
        return ('requests missmatch, expected:\n' +
            text_indent(str(self.expected), '    ') +
            '\n  got:\n' + text_indent(str(self.actual), '    '))

class HTTPServer(HTTPServer_):
    allow_reuse_address = True

    def handle_error(self, request, client_address):
        _exc_class, exc, _tb = sys.exc_info()
        if isinstance(exc, socket.error):
            if exc.errno == errno.EPIPE:
                # suppres 'Broken pipe' errors raised in timeout tests
                return
        HTTPServer_.handle_error(self, request, client_address)

class ThreadedStopableHTTPServer(threading.Thread):
    def __init__(self, address, requests_responses, unordered=False, query_comparator=None):
        threading.Thread.__init__(self, **{'group': None})
        self.requests_responses = requests_responses
        self.daemon = True
        self.sucess = False
        self.shutdown = False
        self.httpd = HTTPServer(address,mock_http_handler(requests_responses,
            unordered=unordered, query_comparator=query_comparator))
        self.httpd.timeout = 1.0
        self.assertions = self.httpd.assertions = []

    @property
    def http_port(self):
        return self.httpd.socket.getsockname()[1]

    def run(self):
        while self.requests_responses:
            if self.shutdown: break
            self.httpd.handle_request()
        if self.requests_responses:
            missing_req = [req for req, resp in self.requests_responses]
            self.assertions.append(
                RequestError('missing requests: ' + ','.join(map(str, missing_req)))
            )
        if not self.assertions:
            self.sucess = True
        # force socket close so next test can bind to same address
        self.httpd.socket.close()

class ThreadedSingleRequestHTTPServer(threading.Thread):
    def __init__(self, address, request_handler):
        threading.Thread.__init__(self, **{'group': None})
        self.daemon = True
        self.sucess = False
        self.shutdown = False
        self.httpd = HTTPServer(address, request_handler)
        self.httpd.timeout = 1.0
        self.assertions = self.httpd.assertions = []

    def run(self):
        self.httpd.handle_request()
        if not self.assertions:
            self.sucess = True
        # force socket close so next test can bind to same address
        self.httpd.socket.close()


def mock_http_handler(requests_responses, unordered=False, query_comparator=None):
    if query_comparator is None:
        query_comparator = query_eq
    class MockHTTPHandler(BaseHTTPRequestHandler):
        def do_GET(self):
            self.query_data = self.path
            return self.do_mock_request('GET')

        def do_POST(self):
            length = int(self.headers['content-length'])
            self.query_data = self.path + '?' + self.rfile.read(length).decode('utf-8')
            return self.do_mock_request('POST')

        def _matching_req_resp(self):
            if len(requests_responses) == 0:
                return None, None
            if unordered:
                for req_resp in requests_responses:
                    req, resp = req_resp
                    if query_comparator(req['path'], self.query_data):
                        requests_responses.remove(req_resp)
                        return req, resp
                return None, None
            else:
                return requests_responses.pop(0)

        def do_mock_request(self, method):
            req, resp = self._matching_req_resp()
            if not req:
                self.server.assertions.append(
                    RequestError('got unexpected request: %s' % self.query_data)
                )
                return
            if 'method' in req:
                if req['method'] != method:
                    self.server.assertions.append(
                        RequestMissmatch('unexpected method', req['method'], method)
                    )
                    self.server.shutdown = True
            if req.get('require_basic_auth', False):
                if 'Authorization' not in self.headers:
                    requests_responses.insert(0, (req, resp)) # push back
                    self.send_response(401)
                    self.send_header('WWW-Authenticate', 'Basic realm="Secure Area"')
                    self.end_headers()
                    self.wfile.write(b'no access')
                    return
            if req.get('headers'):
                for k, v in req['headers'].items():
                    if k not in self.headers:
                        self.server.assertions.append(
                            RequestMissmatch('missing header', k, self.headers)
                        )
                    elif self.headers[k] != v:
                        self.server.assertions.append(
                            RequestMissmatch('header missmatch', '%s: %s' % (k, v), self.headers)
                        )
            if not query_comparator(req['path'], self.query_data):
                self.server.assertions.append(
                    RequestMissmatch('requests differ', req['path'], self.query_data)
                )
                query_actual = set(query_to_dict(self.query_data).items())
                query_expected = set(query_to_dict(req['path']).items())
                self.server.assertions.append(
                    RequestMissmatch('requests params differ', query_expected - query_actual, query_actual - query_expected)
                )
                self.server.shutdown = True
            if 'req_assert_function' in req:
                if not req['req_assert_function'](self):
                    self.server.assertions.append(
                        RequestError('req_assert_function failed')
                    )
                    self.server.shutdown = True
            if 'duration' in resp:
                time.sleep(float(resp['duration']))
            self.start_response(resp)
            if resp.get('body_file'):
                with open(resp['body_file'], 'rb') as f:
{'fraction_non_alphanumeric': 0.0915231131490891, 'fraction_numerical': 0.003098277357789069, 'mean_word_length': 2.077025738798856, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2932963', 'n_tokens_mistral': 4373, 'n_tokens_neox': 4256, 'n_words': 1094}