<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-



import io
import json
import time
import os
import sys
import unittest

import requests
from requests.compat import str, bytes, StringIO
# import envoy
from requests import HTTPError
from requests import get, post, head, put
from requests.auth import HTTPBasicAuth, HTTPDigestAuth



if (sys.platform == 'win32') and ('HTTPBIN_URL' not in os.environ):
    os.environ['HTTPBIN_URL'] = 'http://httpbin.org/'

# TODO: Detect an open port.
PORT = os.environ.get('HTTPBIN_PORT', '7077')
HTTPBIN_URL = os.environ.get('HTTPBIN_URL', 'http://0.0.0.0:%s/' % (PORT))


def httpbin(*suffix):
    """Returns url for HTTPBIN resource."""

    return HTTPBIN_URL + '/'.join(suffix)


SERVICES = (httpbin, )

_httpbin = False

class TestSetup(object):
    """Requests test cases."""

    # It goes to eleven.
    _multiprocess_can_split_ = True

    def setUp(self):

        global _httpbin

        if (not 'HTTPBIN_URL' in os.environ) and not _httpbin:
            # c = envoy.connect('httpbin %s' % (PORT))
            # time.sleep(1)
            _httpbin = True


class RequestsTestSuite(TestSetup, unittest.TestCase):
    """Requests test cases."""
    
    def test_entry_points(self):

        requests.session
        requests.session().get
        requests.session().head
        requests.get
        requests.head
        requests.put
        requests.patch
        requests.post



    def test_invalid_url(self):
        self.assertRaises(ValueError, get, 'hiwpefhipowhefopw')


    def test_path_is_not_double_encoded(self):
        request = requests.Request("http://0.0.0.0/get/test case")

        assert request.path_url == "/get/test%20case"

    def test_HTTP_200_OK_GET(self):
        r = get(httpbin('/get'))
        self.assertEqual(r.status_code, 200)

    def test_response_sent(self):
        r = get(httpbin('/get'))

        self.assertTrue(r.request.sent)

    def test_HTTP_302_ALLOW_REDIRECT_GET(self):
        r = get(httpbin('redirect', '1'))
        self.assertEqual(r.status_code, 200)

    def test_HTTP_302_GET(self):
        r = get(httpbin('redirect', '1'), allow_redirects=False)
        self.assertEqual(r.status_code, 302)


    def test_HTTP_200_OK_GET_WITH_PARAMS(self):
        heads = {'User-agent': 'Mozilla/5.0'}

        r = get(httpbin('user-agent'), headers=heads)

        assert heads['User-agent'] in r.text
        self.assertEqual(r.status_code, 200)


    def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self):
        heads = {'User-agent': 'Mozilla/5.0'}

        r = get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads)
        self.assertEqual(r.status_code, 200)

    def test_session_with_unicode_headers(self):
        heads = { u'User-Agent': u'\u30cd\u30c3\u30c8\u30ef\u30fc\u30af' }

        requests.get(url=httpbin('get'), headers=heads)


    def test_session_with_escaped_url(self):
        # Test a URL that contains percent-escaped characters
        # This URL should not be modified (double-escaped)
        # Tests:
        # - Quoted illegal characters ("%20" (' '), "%3C" ('<'), "%3E" ('>'))
        # - Quoted reserved characters ("%25" ('%'), "%23" ('#'), "%2F" ('/'))
        # - Quoted non-ASCII characters ("%C3%98", "%C3%A5")
        path_fully_escaped = '%3Ca%25b%23c%2Fd%3E/%C3%98%20%C3%A5'
        url = httpbin('get/' + path_fully_escaped)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/' + path_fully_escaped))

        # Test that illegal characters in a path get properly percent-escaped
        # Tests:
        # - Bare illegal characters (space, '<')
        # - Bare non-ASCII characters ('\u00d8')
        path = u'<a%25b%23c%2Fd%3E/\u00d8 %C3%A5'
        url = httpbin('get/' + path)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/' + path_fully_escaped))

        # Test that reserved characters in a path do not get percent-escaped
        # Tests:
        # - All reserved characters (RFC 3986), except '?', '#', '[' and ']',
        #   which are not allowed in the path, and ';' which delimits
        #   parameters.
        #   All such characters must be allowed bare in path, and must not be
        #   encoded.
        # - Special unreserved characters (RFC 3986), which should not be
        #   encoded (even though it wouldn't hurt).
        path_reserved = '!$&\'()*+,/:=@-._~'
        url = httpbin('get/' + path_reserved)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/' + path_reserved))

        # Test that percent-encoded unreserved characters in a path get
        # normalised to their un-encoded forms.
        path_unreserved = 'ABCDwxyz1234-._~'
        path_unreserved_escaped = '%41%42%43%44%77%78%79%7A%31%32%33%34%2D%2E%5F%7E'
        url = httpbin('get/' + path_unreserved_escaped)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/' + path_unreserved))


    def test_user_agent_transfers(self):
        """Issue XX"""

        heads = {
            'User-agent':
                'Mozilla/5.0 (github.com/kennethreitz/requests)'
        }

        r = get(httpbin('user-agent'), headers=heads);
        self.assertTrue(heads['User-agent'] in r.text)

        heads = {
            'user-agent':
                'Mozilla/5.0 (github.com/kennethreitz/requests)'
        }

        r = get(httpbin('user-agent'), headers=heads);
        self.assertTrue(heads['user-agent'] in r.text)


    def test_HTTP_200_OK_HEAD(self):
        r = head(httpbin('/get'))
        self.assertEqual(r.status_code, 200)<commit_msg>Added test cases for URI cleaning in the query part (Issue #429).<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-



import io
import json
import time
import os
import sys
import unittest

import requests
from requests.compat import str, bytes, StringIO
# import envoy
from requests import HTTPError
from requests import get, post, head, put
from requests.auth import HTTPBasicAuth, HTTPDigestAuth



if (sys.platform == 'win32') and ('HTTPBIN_URL' not in os.environ):
    os.environ['HTTPBIN_URL'] = 'http://httpbin.org/'

# TODO: Detect an open port.
PORT = os.environ.get('HTTPBIN_PORT', '7077')
HTTPBIN_URL = os.environ.get('HTTPBIN_URL', 'http://0.0.0.0:%s/' % (PORT))


def httpbin(*suffix):
    """Returns url for HTTPBIN resource."""

    return HTTPBIN_URL + '/'.join(suffix)


SERVICES = (httpbin, )

_httpbin = False

class TestSetup(object):
    """Requests test cases."""

    # It goes to eleven.
    _multiprocess_can_split_ = True

    def setUp(self):

        global _httpbin

        if (not 'HTTPBIN_URL' in os.environ) and not _httpbin:
            # c = envoy.connect('httpbin %s' % (PORT))
            # time.sleep(1)
            _httpbin = True


class RequestsTestSuite(TestSetup, unittest.TestCase):
    """Requests test cases."""
    
    def test_entry_points(self):

        requests.session
        requests.session().get
        requests.session().head
        requests.get
        requests.head
        requests.put
        requests.patch
        requests.post



    def test_invalid_url(self):
        self.assertRaises(ValueError, get, 'hiwpefhipowhefopw')


    def test_path_is_not_double_encoded(self):
        request = requests.Request("http://0.0.0.0/get/test case")

        assert request.path_url == "/get/test%20case"

    def test_HTTP_200_OK_GET(self):
        r = get(httpbin('/get'))
        self.assertEqual(r.status_code, 200)

    def test_response_sent(self):
        r = get(httpbin('/get'))

        self.assertTrue(r.request.sent)

    def test_HTTP_302_ALLOW_REDIRECT_GET(self):
        r = get(httpbin('redirect', '1'))
        self.assertEqual(r.status_code, 200)

    def test_HTTP_302_GET(self):
        r = get(httpbin('redirect', '1'), allow_redirects=False)
        self.assertEqual(r.status_code, 302)


    def test_HTTP_200_OK_GET_WITH_PARAMS(self):
        heads = {'User-agent': 'Mozilla/5.0'}

        r = get(httpbin('user-agent'), headers=heads)

        assert heads['User-agent'] in r.text
        self.assertEqual(r.status_code, 200)


    def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self):
        heads = {'User-agent': 'Mozilla/5.0'}

        r = get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads)
        self.assertEqual(r.status_code, 200)

    def test_session_with_unicode_headers(self):
        heads = { u'User-Agent': u'\u30cd\u30c3\u30c8\u30ef\u30fc\u30af' }

        requests.get(url=httpbin('get'), headers=heads)


    def test_session_with_escaped_url(self):
        # Test a URL that contains percent-escaped characters
        # This URL should not be modified (double-escaped)
        # Tests:
        # - Quoted illegal characters ("%20" (' '), "%3C" ('<'), "%3E" ('>'))
        # - Quoted reserved characters ("%25" ('%'), "%23" ('#'), "%2F" ('/'))
        # - Quoted non-ASCII characters ("%C3%98", "%C3%A5")
        path_fully_escaped = '%3Ca%25b%23c%2Fd%3E/%C3%98%20%C3%A5'
        url = httpbin('get/' + path_fully_escaped)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/' + path_fully_escaped))

        # Test that illegal characters in a path get properly percent-escaped
        # Tests:
        # - Bare illegal characters (space, '<')
        # - Bare non-ASCII characters ('\u00d8')
        path = u'<a%25b%23c%2Fd%3E/\u00d8 %C3%A5'
        url = httpbin('get/' + path)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/' + path_fully_escaped))

        # Test that reserved characters in a path do not get percent-escaped
        # Tests:
        # - All reserved characters (RFC 3986), except '?', '#', '[' and ']',
        #   which are not allowed in the path, and ';' which delimits
        #   parameters.
        #   All such characters must be allowed bare in path, and must not be
        #   encoded.
        # - Special unreserved characters (RFC 3986), which should not be
        #   encoded (even though it wouldn't hurt).
        path_reserved = '!$&\'()*+,/:=@-._~'
        url = httpbin('get/' + path_reserved)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/' + path_reserved))

        # Test that percent-encoded unreserved characters in a path get
        # normalised to their un-encoded forms.
        path_unreserved = 'ABCDwxyz1234-._~'
        path_unreserved_escaped = '%41%42%43%44%77%78%79%7A%31%32%33%34%2D%2E%5F%7E'
        url = httpbin('get/' + path_unreserved_escaped)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/' + path_unreserved))

        # Re-run all of the same tests on the query part of the URI
        query_fully_escaped = '%3Ca%25b%23c%2Fd%3E=%C3%98%20%C3%A5'
        url = httpbin('get/?' + query_fully_escaped)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/?' + query_fully_escaped))

        query = u'<a%25b%23c%2Fd%3E=\u00d8 %C3%A5'
        url = httpbin('get/?' + query)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/?' + query_fully_escaped))

        # The legal characters in query happens to be the same as in path
        query_reserved = '!$&\'()*+,/:=@-._~'
        url = httpbin('get/?' + query_reserved)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/?' + query_reserved))

        query_unreserved = 'ABCDwxyz=1234-._~'
        query_unreserved_escaped = '%41%42%43%44%77%78%79%7A=%31%32%33%34%2D%2E%5F%7E'
        url = httpbin('get/?' + query_unreserved_escaped)
        response = get(url)
        self.assertEqual(response.url, httpbin('get/?' + query_unreserved))


    def test_user_agent_transfers(self):
        """Issue XX"""

        heads = {
            'User-agent':
                'Mozilla/5.0 (github.com/kennethreitz/requests)'
        }

        r = get(httpbin('user-agent'), headers=heads);
        self.assertTrue(heads['User-agent'] in r.text)

        heads = {
            'user-agent':
                'Mozilla/5.0 (github.com/kennethreitz/requests)'
        }

        r = get(httpbin('user-agent'), headers=heads);
        self.assertTrue(heads['user-agent'] in r.text)


    def test_HTTP_200_OK_HEAD(self):
        r = head(httpbin('/get'))
        self.assertEqual(r.status_code, 200)
{'fraction_non_alphanumeric': 0.1420376504807304, 'fraction_numerical': 0.030459723681021247, 'mean_word_length': 2.678454680534918, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2832888', 'n_tokens_mistral': 4382, 'n_tokens_neox': 4064, 'n_words': 868}