<commit_before>"""Fork of urllib2.

When reading this, don't assume that all code in here is reachable.  Code in
the rest of mechanize may be used instead.

Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Python
Software Foundation; All Rights Reserved

Copyright 2002-2009 <NAME> <<EMAIL>>

This code is free software; you can redistribute it and/or modify it
under the terms of the BSD or ZPL 2.1 licenses (see the file
COPYING.txt included with the distribution).

"""

# XXX issues:
# If an authentication error handler that tries to perform
# authentication for some reason but fails, how should the error be
# signalled?  The client needs to know the HTTP error code.  But if
# the handler knows that the problem was, e.g., that it didn't know
# that hash algo that requested in the challenge, it would be good to
# pass that information along to the client, too.
# ftp errors aren't handled cleanly
# check digest against correct (i.e. non-apache) implementation

# Possible extensions:
# complex proxies  XXX not sure what exactly was meant by this
# abstract factory for opener

from __future__ import absolute_import

import base64
import bisect
import copy
import hashlib
import logging
import os
import platform
import posixpath
import re
import socket
import sys
import time
from collections import OrderedDict
from functools import partial
from io import BytesIO, BufferedReader

from . import _rfc3986
from ._clientcookie import CookieJar
from ._headersutil import normalize_header_name
from ._response import closeable_response
from .polyglot import (HTTPConnection, HTTPError, HTTPSConnection, URLError,
                       as_unicode, create_response_info, ftpwrapper,
                       getproxies, is_class, is_mapping, is_py2, is_string,
                       iteritems, map, raise_with_traceback, splitattr,
                       splitpasswd, splitport, splittype, splituser,
                       splitvalue, unquote, unwrap, url2pathname,
                       urllib_proxy_bypass, urllib_splithost, urlparse,
                       urlsplit)


def sha1_digest(data):
    if not isinstance(data, bytes):
        data = data.encode('utf-8')
    return hashlib.sha1(data).hexdigest()


def md5_digest(data):
    if not isinstance(data, bytes):
        data = data.encode('utf-8')
    return hashlib.md5(data).hexdigest()


if platform.python_implementation() == 'PyPy':
    def create_readline_wrapper(fh):
        fh.recv = fh.read
        if not hasattr(fh, '_drop'):
            fh._drop = lambda: None
            fh._reuse = lambda: None
        return socket._fileobject(fh, close=True)
else:
    def create_readline_wrapper(fh):
        fh.recv = fh.read
        if is_py2:
            ans = socket._fileobject(fh, close=True)
        else:
            fh.recv_into = fh.readinto
            fh._decref_socketios = lambda: None
            ans = BufferedReader(socket.SocketIO(fh, 'r'))
        return ans


splithost = urllib_splithost


# used in User-Agent header sent
__version__ = sys.version[:3]

_opener = None


def urlopen(url, data=None):
    global _opener
    if _opener is None:
        _opener = build_opener()
    return _opener._open(url, data)


def install_opener(opener):
    global _opener
    _opener = opener


# copied from cookielib.py
_cut_port_re = re.compile(r":\d+$")


def request_host(request):
    """Return request-host, as defined by RFC 2965.

    Variation from RFC: returned value is lowercased, for convenient
    comparison.

    """
    url = request.get_full_url()
    host = urlparse(url)[1]
    if host == "":
        host = request.get_header("Host", "")

    # remove port, if present
    host = _cut_port_re.sub("", host, 1)
    return host.lower()


class Request:

    def __init__(self, url, data=None, headers={},
                 origin_req_host=None, unverifiable=False, method=None):
        # unwrap('<URL:type://host/path>') --> 'type://host/path'
        self.__original = unwrap(url)
        self.type = None
        self._method = method and str(method)
        # self.__r_type is what's left after doing the splittype
        self.host = None
        self.port = None
        self._tunnel_host = None
        self.data = data
        self.headers = OrderedDict()
        for key, value in iteritems(headers):
            self.add_header(key, value)
        self.unredirected_hdrs = OrderedDict()
        if origin_req_host is None:
            origin_req_host = request_host(self)
        self.origin_req_host = origin_req_host
        self.unverifiable = unverifiable
        try:
            self.get_host()  # in py3 cookiejar expect self.host to be not None
        except Exception:
            self.host = None

    def __getattr__(self, attr):
        # XXX this is a fallback mechanism to guard against these
        # methods getting called in a non-standard order.  this may be
        # too complicated and/or unnecessary.
        # XXX should the __r_XXX attributes be public?
        if attr[:12] == '_Request__r_':
            name = attr[12:]
            if hasattr(Request, 'get_' + name):
                getattr(self, 'get_' + name)()
                return getattr(self, attr)
        raise AttributeError(attr)

    def get_method(self):
        ' The method used for HTTP requests '
        if self._method is None:
            return "POST" if self.has_data() else 'GET'
        return self._method

    # XXX these helper methods are lame

    def set_data(self, data):
        ' Set the data (a bytestring) to be sent with this request '
        self.data = data
    add_data = set_data

    def has_data(self):
        ' True iff there is some data to be sent with this request '
        return self.data is not None

    def get_data(self):
        ' The data to be sent with this request '
        return self.data

    def get_full_url(self):
        return self.__original

    def get_type(self):
        if self.type is None:
            self.type, self.__r_type = splittype(self.__original)
            if self.type is None:
                raise ValueError("unknown url type: %s" % self.__original)
        return self.type

    def get_host(self):
        if self.host is None:
            self.host, self.__r_host = splithost(self.__r_type)
            if self.host:
                self.host = unquote(self.host)
        return self.host

    def get_selector(self):
        scheme, authority, path, query, fragment = _rfc3986.urlsplit(
            self.__r_host)
        if path == "":
            path = "/"  # RFC 2616, section 3.2.2
        fragment = None  # RFC 3986, section 3.5
        return _rfc3986.urlunsplit([scheme, authority, path, query, fragment])

    def set_proxy(self, host, type):<commit_msg>Add a full_request property to the Request class<commit_after>"""Fork of urllib2.

When reading this, don't assume that all code in here is reachable.  Code in
the rest of mechanize may be used instead.

Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Python
Software Foundation; All Rights Reserved

Copyright 2002-2009 <NAME> <<EMAIL>>

This code is free software; you can redistribute it and/or modify it
under the terms of the BSD or ZPL 2.1 licenses (see the file
COPYING.txt included with the distribution).

"""

# XXX issues:
# If an authentication error handler that tries to perform
# authentication for some reason but fails, how should the error be
# signalled?  The client needs to know the HTTP error code.  But if
# the handler knows that the problem was, e.g., that it didn't know
# that hash algo that requested in the challenge, it would be good to
# pass that information along to the client, too.
# ftp errors aren't handled cleanly
# check digest against correct (i.e. non-apache) implementation

# Possible extensions:
# complex proxies  XXX not sure what exactly was meant by this
# abstract factory for opener

from __future__ import absolute_import

import base64
import bisect
import copy
import hashlib
import logging
import os
import platform
import posixpath
import re
import socket
import sys
import time
from collections import OrderedDict
from functools import partial
from io import BytesIO, BufferedReader

from . import _rfc3986
from ._clientcookie import CookieJar
from ._headersutil import normalize_header_name
from ._response import closeable_response
from .polyglot import (HTTPConnection, HTTPError, HTTPSConnection, URLError,
                       as_unicode, create_response_info, ftpwrapper,
                       getproxies, is_class, is_mapping, is_py2, is_string,
                       iteritems, map, raise_with_traceback, splitattr,
                       splitpasswd, splitport, splittype, splituser,
                       splitvalue, unquote, unwrap, url2pathname,
                       urllib_proxy_bypass, urllib_splithost, urlparse,
                       urlsplit)


def sha1_digest(data):
    if not isinstance(data, bytes):
        data = data.encode('utf-8')
    return hashlib.sha1(data).hexdigest()


def md5_digest(data):
    if not isinstance(data, bytes):
        data = data.encode('utf-8')
    return hashlib.md5(data).hexdigest()


if platform.python_implementation() == 'PyPy':
    def create_readline_wrapper(fh):
        fh.recv = fh.read
        if not hasattr(fh, '_drop'):
            fh._drop = lambda: None
            fh._reuse = lambda: None
        return socket._fileobject(fh, close=True)
else:
    def create_readline_wrapper(fh):
        fh.recv = fh.read
        if is_py2:
            ans = socket._fileobject(fh, close=True)
        else:
            fh.recv_into = fh.readinto
            fh._decref_socketios = lambda: None
            ans = BufferedReader(socket.SocketIO(fh, 'r'))
        return ans


splithost = urllib_splithost


# used in User-Agent header sent
__version__ = sys.version[:3]

_opener = None


def urlopen(url, data=None):
    global _opener
    if _opener is None:
        _opener = build_opener()
    return _opener._open(url, data)


def install_opener(opener):
    global _opener
    _opener = opener


# copied from cookielib.py
_cut_port_re = re.compile(r":\d+$")


def request_host(request):
    """Return request-host, as defined by RFC 2965.

    Variation from RFC: returned value is lowercased, for convenient
    comparison.

    """
    url = request.get_full_url()
    host = urlparse(url)[1]
    if host == "":
        host = request.get_header("Host", "")

    # remove port, if present
    host = _cut_port_re.sub("", host, 1)
    return host.lower()


class Request:

    def __init__(self, url, data=None, headers={},
                 origin_req_host=None, unverifiable=False, method=None):
        # unwrap('<URL:type://host/path>') --> 'type://host/path'
        self.__original = unwrap(url)
        self.type = None
        self._method = method and str(method)
        # self.__r_type is what's left after doing the splittype
        self.host = None
        self.port = None
        self._tunnel_host = None
        self.data = data
        self.headers = OrderedDict()
        for key, value in iteritems(headers):
            self.add_header(key, value)
        self.unredirected_hdrs = OrderedDict()
        if origin_req_host is None:
            origin_req_host = request_host(self)
        self.origin_req_host = origin_req_host
        self.unverifiable = unverifiable
        try:
            self.get_host()  # in py3 cookiejar expect self.host to be not None
        except Exception:
            self.host = None

    def __getattr__(self, attr):
        # XXX this is a fallback mechanism to guard against these
        # methods getting called in a non-standard order.  this may be
        # too complicated and/or unnecessary.
        # XXX should the __r_XXX attributes be public?
        if attr[:12] == '_Request__r_':
            name = attr[12:]
            if hasattr(Request, 'get_' + name):
                getattr(self, 'get_' + name)()
                return getattr(self, attr)
        raise AttributeError(attr)

    def get_method(self):
        ' The method used for HTTP requests '
        if self._method is None:
            return "POST" if self.has_data() else 'GET'
        return self._method

    # XXX these helper methods are lame

    def set_data(self, data):
        ' Set the data (a bytestring) to be sent with this request '
        self.data = data
    add_data = set_data

    def has_data(self):
        ' True iff there is some data to be sent with this request '
        return self.data is not None

    def get_data(self):
        ' The data to be sent with this request '
        return self.data

    def get_full_url(self):
        return self.__original

    @property
    def full_url(self):
        # In python 3 this is a deleteable and settable property, which when
        # deleted gets set to None. But this interface does not seem to be used
        # by any stdlib code, so this should be sufficient.
        return self.__original

    def get_type(self):
        if self.type is None:
            self.type, self.__r_type = splittype(self.__original)
            if self.type is None:
                raise ValueError("unknown url type: %s" % self.__original)
        return self.type

    def get_host(self):
        if self.host is None:
            self.host, self.__r_host = splithost(self.__r_type)
            if self.host:
                self.host = unquote(self.host)
        return self.host

    def get_selector(self):
        scheme, authority, path, query, fragment = _rfc3986.urlsplit(
            self.__r_host)
        if path == "":
            path = "/"  # RFC 2616, section 3.2.2
        fragment = None  # RFC 3986, section 3.5
        return _rfc3986.urlunsplit([scheme, authority, path, query, fragment])

    def set_proxy(self, host, type):
{'fraction_non_alphanumeric': 0.0818418013856813, 'fraction_numerical': 0.013784642032332563, 'mean_word_length': 2.48779260005034, 'pattern_counts': {'":': 6, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4090772', 'n_tokens_mistral': 4250, 'n_tokens_neox': 3965, 'n_words': 1372}