<commit_before># Copyright (c) 2012-2013 <NAME> http://garnaat.org/
# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import os
import logging
import time
import threading

from botocore.vendored.requests.sessions import Session
from botocore.vendored.requests.utils import get_environ_proxies
from botocore.vendored import six

from botocore.exceptions import UnknownEndpointError
from botocore.awsrequest import AWSRequest
from botocore.compat import filter_ssl_san_warnings, urlsplit
from botocore.compat import urlunsplit
from botocore.utils import percent_encode_sequence, normalize_url_path
from botocore.hooks import first_non_none_response
from botocore.response import StreamingBody
from botocore import parsers


logger = logging.getLogger(__name__)
DEFAULT_TIMEOUT = 60
NOT_SET = object()
filter_ssl_san_warnings()


def convert_to_response_dict(http_response, operation_model):
    """Convert an HTTP response object to a request dict.

    This converts the requests library's HTTP response object to
    a dictionary.

    :type http_response: botocore.vendored.requests.model.Response
    :param http_response: The HTTP response from an AWS service request.

    :rtype: dict
    :return: A response dictionary which will contain the following keys:
        * headers (dict)
        * status_code (int)
        * body (string or file-like object)

    """
    response_dict = {
        'headers': http_response.headers,
        'status_code': http_response.status_code,
    }
    if response_dict['status_code'] >= 300:
        response_dict['body'] = http_response.content
    elif operation_model.has_streaming_output:
        response_dict['body'] = StreamingBody(
            http_response.raw, response_dict['headers'].get('content-length'))
    else:
        response_dict['body'] = http_response.content
    return response_dict


class PreserveAuthSession(Session):
    def rebuild_auth(self, prepared_request, response):
        pass


class RequestCreator(object):
    """This object can convert a request dict to an AWSRequest class."""

    def create_request_object(self, request_dict, user_agent, endpoint_url):
        """

        :type request_dict: dict
        :param request_dict:  The request dict (created from the ``serialize``
            module).

        :type user_agent: string
        :param user_agent: The user agent to use for this request.

        :type endpoint_url: string
        :param endpoint_url: The full endpoint url, which contains at least
            the scheme, the hostname, and optionally any path components.

        :rtype: ``botocore.awsrequest.AWSRequest``
        :return: An AWSRequest object based on the request_dict.

        """
        r = request_dict
        headers = r['headers']
        headers['User-Agent'] = user_agent
        url = self._urljoin(endpoint_url, r['url_path'])
        if r['query_string']:
            encoded_query_string = percent_encode_sequence(r['query_string'])
            if '?' not in url:
                url += '?%s' % encoded_query_string
            else:
                url += '&%s' % encoded_query_string
        request = AWSRequest(method=r['method'], url=url,
                             data=r['body'],
                             headers=headers)
        return request

    def _urljoin(self, endpoint_url, url_path):
        p = urlsplit(endpoint_url)
        # <part>   - <index>
        # scheme   - p[0]
        # netloc   - p[1]
        # path     - p[2]
        # query    - p[3]
        # fragment - p[4]
        if not url_path or url_path == '/':
            # If there's no path component, ensure the URL ends with
            # a '/' for backwards compatibility.
            if not p[2]:
                return endpoint_url + '/'
            return endpoint_url
        new_path = normalize_url_path(p[2] + url_path)
        reconstructed = urlunsplit((p[0], p[1], new_path, p[3], p[4]))<commit_msg>Fix failing integ test<commit_after># Copyright (c) 2012-2013 <NAME> http://garnaat.org/
# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import os
import logging
import time
import threading

from botocore.vendored.requests.sessions import Session
from botocore.vendored.requests.utils import get_environ_proxies
from botocore.vendored import six

from botocore.exceptions import UnknownEndpointError
from botocore.awsrequest import AWSRequest
from botocore.compat import filter_ssl_san_warnings, urlsplit
from botocore.compat import urlunsplit
from botocore.utils import percent_encode_sequence
from botocore.hooks import first_non_none_response
from botocore.response import StreamingBody
from botocore import parsers


logger = logging.getLogger(__name__)
DEFAULT_TIMEOUT = 60
NOT_SET = object()
filter_ssl_san_warnings()


def convert_to_response_dict(http_response, operation_model):
    """Convert an HTTP response object to a request dict.

    This converts the requests library's HTTP response object to
    a dictionary.

    :type http_response: botocore.vendored.requests.model.Response
    :param http_response: The HTTP response from an AWS service request.

    :rtype: dict
    :return: A response dictionary which will contain the following keys:
        * headers (dict)
        * status_code (int)
        * body (string or file-like object)

    """
    response_dict = {
        'headers': http_response.headers,
        'status_code': http_response.status_code,
    }
    if response_dict['status_code'] >= 300:
        response_dict['body'] = http_response.content
    elif operation_model.has_streaming_output:
        response_dict['body'] = StreamingBody(
            http_response.raw, response_dict['headers'].get('content-length'))
    else:
        response_dict['body'] = http_response.content
    return response_dict


class PreserveAuthSession(Session):
    def rebuild_auth(self, prepared_request, response):
        pass


class RequestCreator(object):
    """This object can convert a request dict to an AWSRequest class."""

    def create_request_object(self, request_dict, user_agent, endpoint_url):
        """

        :type request_dict: dict
        :param request_dict:  The request dict (created from the ``serialize``
            module).

        :type user_agent: string
        :param user_agent: The user agent to use for this request.

        :type endpoint_url: string
        :param endpoint_url: The full endpoint url, which contains at least
            the scheme, the hostname, and optionally any path components.

        :rtype: ``botocore.awsrequest.AWSRequest``
        :return: An AWSRequest object based on the request_dict.

        """
        r = request_dict
        headers = r['headers']
        headers['User-Agent'] = user_agent
        url = self._urljoin(endpoint_url, r['url_path'])
        if r['query_string']:
            encoded_query_string = percent_encode_sequence(r['query_string'])
            if '?' not in url:
                url += '?%s' % encoded_query_string
            else:
                url += '&%s' % encoded_query_string
        request = AWSRequest(method=r['method'], url=url,
                             data=r['body'],
                             headers=headers)
        return request

    def _urljoin(self, endpoint_url, url_path):
        p = urlsplit(endpoint_url)
        # <part>   - <index>
        # scheme   - p[0]
        # netloc   - p[1]
        # path     - p[2]
        # query    - p[3]
        # fragment - p[4]
        if not url_path or url_path == '/':
            # If there's no path component, ensure the URL ends with
            # a '/' for backwards compatibility.
            if not p[2]:
                return endpoint_url + '/'
            return endpoint_url
        if p[2].endswith('/') and url_path.startswith('/'):
            new_path = p[2][:-1] + url_path
        else:
            new_path = p[2] + url_path
        reconstructed = urlunsplit((p[0], p[1], new_path, p[3], p[4]))
{'fraction_non_alphanumeric': 0.08840209051484488, 'fraction_numerical': 0.008339819859891026, 'mean_word_length': 2.8272340425531914, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2443846', 'n_tokens_mistral': 2569, 'n_tokens_neox': 2487, 'n_words': 797}