import email parser import email message import errno import http import io import re import socket import sys import collections abc from urllib parse import urlsplit httpmessage parseheaders and the http status code constants are intentionally omitted for simplicity all httpresponse httpconnection httpexception notconnected unknownprotocol unknowntransferencoding unimplementedfilemode incompleteread invalidurl improperconnectionstate cannotsendrequest cannotsendheader responsenotready badstatusline linetoolong remotedisconnected error responses httpport 80 httpsport 443 unknown unknown connection states csidle idle csreqstarted requeststarted csreqsent requestsent hack to maintain backwards compatibility globals updatehttp httpstatus members another hack to maintain backwards compatibility mapping status codes to official w3c names responses v v phrase for v in http httpstatus members values maximal line length when calling readline maxline 65536 maxheaders 100 header namevalue abnf http tools ietf orghtmlrfc7230section3 2 vchar x217e obstext x80ff headerfield fieldname ows fieldvalue ows fieldname token fieldvalue fieldcontent obsfold fieldcontent fieldvchar 1 sp htab fieldvchar fieldvchar vchar obstext obsfold crlf 1 sp htab obsolete line folding see section 3 2 4 token 1tchar tchar digit alpha any vchar except delimiters vchar defined in http tools ietf orghtmlrfc5234appendixb 1 the patterns for both name and value are more lenient than rfc definitions to allow for backwards compatibility islegalheadername re compilerb s rn fullmatch isillegalheadervalue re compilerb n tr tn search these characters are not allowed within http url paths see https tools ietf orghtmlrfc3986section3 3 and the https tools ietf orghtmlrfc3986appendixa pchar definition prevents cve20199740 includes control characters such as rn we don t restrict chars above x7f as putrequest limits us to ascii containsdisallowedurlpcharre re compile x00x20x7f arguably only these should allowed isallowedurlpcharsre re compiler azaz09 we are more lenient for assumed real world compatibility purposes these characters are not allowed within http method names to prevent http header injection containsdisallowedmethodpcharre re compile x00x1f we always set the contentlength header for these methods because some servers will otherwise respond with a 411 methodsexpectingbody patch post put def encodedata name data remove interface scope from ipv6 address encname percent encname partitionb if percent assert encname startswithb encname encname b return encname class httpmessageemail message message xxx the only usage of this method is in http server cgihttprequesthandler maybe move the code there so that it doesn t need to be part of the public api the api has never been defined so this could cause backwards compatibility issues def getallmatchingheadersself name name name lower n lenname lst hit 0 for line in self keys if line n lower name hit 1 elif not line 1 isspace hit 0 if hit lst appendline return lst def readheadersfp headers while true line fp readlinemaxline 1 if lenline maxline raise linetoolongheader line headers appendline if lenheaders maxheaders raise httpexceptiongot more than d headers maxheaders if line in b rn b n b break return headers def parseheaderlinesheaderlines classhttpmessage hstring b joinheaderlines decode iso88591 return email parser parserclassclass parsestrhstring def parseheadersfp classhttpmessage see rfc 2616 sec 19 6 and rfc 1945 sec 6 for details the bytes from the socket object are iso88591 strings see rfc 2616 sec 2 2 which notes an exception for mimeencoded text following rfc 2047 the basic status line parsing only accepts iso88591 if the response includes a contentlength header we need to make sure that the client doesn t read more than the specified number of bytes if it does it will block until the server times out and closes the connection this will happen if a self fp read is done without a size whether self fp is buffered or not so no self fp read by clients unless they know what they are doing the httpresponse object is returned via urllib the clients of http and urllib expect different attributes for the headers headers is used here and supports urllib msg is provided as a backwards compatibility layer for http clients from the statusline of the response presumably the server closed the connection before sending a valid response empty version will cause next test to fail the status code is a threedigit number we ve already started reading the response read until we get a non100 response skip the header from the 100 response some servers might still return 0 9 treat it as 1 0 anyway are we using the chunkedstyle of transfer encoding will the connection close at the end of the response do we have a contentlength note rfc 2616 s4 4 3 says we ignore this if trenc is chunked does the body have a fixed length of zero if the connection remains open and we aren t using chunked and a contentlength was not provided then assume that the connection will close an http1 1 proxy is assumed to stay open unless explicitly closed some http1 0 implementations have support for persistent connections using rules different than http1 1 for older http keepalive indicates persistent connection at least akamai returns a connection keepalive header which was supposed to be sent by the client proxyconnection is a netscape hack otherwise assume it will close these implementations are for the benefit of io bufferedreader xxx this class should probably be revised to act more like the raw stream that bufferedreader expects always returns true return true end of raw stream methods def isclosedself note it is possible that we will not ever call self close this case occurs when willclose is true length is none and we read up to the last byte but not past it implies if willclose is false then self close will always be called meaning self isclosed is meaningful read and return the response body or up to the next amt bytes if self fp is none return b if self method head self closeconn return b if self chunked return self readchunkedamt if amt is not none if self length is not none and amt self length clip the read to the end of response amt self length s self fp readamt if not s and amt ideally we would raise incompleteread if the contentlength wasn t satisfied but it might break compatibility self closeconn elif self length is not none self length lens if not self length self closeconn return s else amount is not given unbounded read so we must check self length if self length is none s self fp read else try s self safereadself length except incompleteread self closeconn raise self length 0 self closeconn we read everything return s def readintoself b if self fp is none return 0 if self method head self closeconn return 0 if self chunked return self readintochunkedb if self length is not none if lenb self length clip the read to the end of response b memoryviewb0 self length we do not use saferead here because this may be a willclose connection and the user is reading more bytes than will be provided for example reading in 1k chunks n self fp readintob if not n and b ideally we would raise incompleteread if the contentlength wasn t satisfied but it might break compatibility self closeconn elif self length is not none self length n if not self length self closeconn return n def readnextchunksizeself read the next chunk size from the file line self fp readlinemaxline 1 if lenline maxline raise linetoolongchunk size i line findb if i 0 line line i strip chunkextensions try return intline 16 except valueerror close the connection as protocol synchronisation is probably lost self closeconn raise def readanddiscardtrailerself read and discard trailer up to the crlf terminator note we shouldn t have any trailers while true line self fp readlinemaxline 1 if lenline maxline raise linetoolongtrailer line if not line a vanishingly small number of sites eof without sending the trailer break if line in b rn b n b break def getchunkleftself return self chunkleft reading a new chunk if necessary chunkleft 0 at the end of the current chunk need to close it chunkleft none no current chunk should read next this function returns nonzero or none if the last chunk has been read chunkleft self chunkleft if not chunkleft can be 0 or none if chunkleft is not none we are at the end of chunk discard chunk end self saferead2 toss the crlf at the end of the chunk try chunkleft self readnextchunksize except valueerror raise incompletereadb if chunkleft 0 last chunk 10 chunkextension crlf self readanddiscardtrailer we read everything close the file self closeconn chunkleft none self chunkleft chunkleft return chunkleft def readchunkedself amtnone assert self chunked unknown value try while chunkleft self getchunkleft is not none if amt is not none and amt chunkleft value appendself safereadamt self chunkleft chunkleft amt break value appendself safereadchunkleft if amt is not none amt chunkleft self chunkleft 0 return b joinvalue except incompleteread as exc raise incompletereadb joinvalue from exc def readintochunkedself b assert self chunked unknown totalbytes 0 mvb memoryviewb try while true chunkleft self getchunkleft if chunkleft is none return totalbytes if lenmvb chunkleft n self safereadintomvb self chunkleft chunkleft n return totalbytes n tempmvb mvb chunkleft n self safereadintotempmvb mvb mvbn totalbytes n self chunkleft 0 except incompleteread raise incompletereadbytesb0 totalbytes def safereadself amt data self fp readamt if lendata amt raise incompletereaddata amtlendata return data def safereadintoself b read with at most one underlying system call if at least one byte is buffered return that instead having this enables iobase readline to read more than one byte at a time fallback to iobase readline which uses peek and read strictly speaking getchunkleft may cause more than one read but that is ok since that is to satisfy the chunked protocol strictly speaking getchunkleft may cause more than one read but that is ok since that is to satisfy the chunked protocol peek is allowed to return more than requested just request the entire chunk and truncate what we get returns the value of the header matching name if there are multiple matching headers the values are combined into a single string separated by commas and spaces if no matching header is found returns default or none if the default is not specified if the headers are unknown raises http client responsenotready return list of header value tuples if self headers is none raise responsenotready return listself headers items we override iobase iter so that it doesn t check for closedness def iterself return self for compatibility with oldstyle urllib responses def infoself return self headers def geturlself return self url def getcodeself return self status def createhttpscontexthttpversion function also used by urllib request to be able to set the checkhostname attribute on a context object context ssl createdefaulthttpscontext send alpn extension to indicate http1 1 protocol if httpversion 11 context setalpnprotocols http1 1 enable pha for tls 1 3 connections if available if context posthandshakeauth is not none context posthandshakeauth true return context class httpconnection httpvsn 11 httpvsnstr http1 1 responseclass httpresponse defaultport httpport autoopen 1 debuglevel 0 staticmethod def istextiostream return isinstancestream io textiobase staticmethod def getcontentlengthbody method if body is none do an explicit check for not none here to distinguish between unset and set but empty if method upper in methodsexpectingbody return 0 else return none if hasattrbody read filelike object return none try does it implement the buffer protocol bytes bytearray array mv memoryviewbody return mv nbytes except typeerror pass if isinstancebody str return lenbody return none def initself host portnone timeoutsocket globaldefaulttimeout sourceaddressnone blocksize8192 self timeout timeout self sourceaddress sourceaddress self blocksize blocksize self sock none self buffer self response none self state csidle self method none self tunnelhost none self tunnelport none self tunnelheaders self rawproxyheaders none self host self port self gethostporthost port self validatehostself host this is stored as an instance variable to allow unit tests to replace it with a suitable mockup self createconnection socket createconnection def settunnelself host portnone headersnone if self sock raise runtimeerrorcan t set up tunnel for established connection self tunnelhost self tunnelport self gethostporthost port if headers self tunnelheaders headers copy else self tunnelheaders clear if not anyheader lower host for header in self tunnelheaders encodedhost self tunnelhost encodeidna decodeascii self tunnelheadershost s d encodedhost self tunnelport def gethostportself host port if port is none i host rfind j host rfind ipv6 addresses have if i j try port inthosti1 except valueerror if hosti1 http foo com http foo com port self defaultport else raise invalidurlnonnumeric port s hosti1 host host i else port self defaultport if host and host0 and host1 host host1 1 return host port def setdebuglevelself level self debuglevel level def tunnelself connect bconnect s d srn self tunnelhost encodeidna self tunnelport self httpvsnstr encodeascii headers connect for header value in self tunnelheaders items headers appendfheader valuern encodelatin1 headers appendbrn making a single send call instead of one per line encourages the host os to use a more optimal packet size instead of potentially emitting a series of small packets self sendb joinheaders del headers response self responseclassself sock methodself method try version code message response readstatus self rawproxyheaders readheadersresponse fp if self debuglevel 0 for header in self rawproxyheaders print header header decode if code http httpstatus ok self close raise oserrorftunnel connection failed code message strip finally response close def getproxyresponseheadersself return parseheaderlinesself rawproxyheaders if self rawproxyheaders is not none else none def connectself might fail in oss that don t implement tcpnodelay close the connection to the http server self state csidle try sock self sock if sock self sock none sock close close it manually there may be other refs finally response self response if response self response none response close def sendself data if self sock is none if self autoopen self connect else raise notconnected if self debuglevel 0 printsend reprdata if hasattrdata read if self debuglevel 0 printsending a readable encode self istextiodata if encode and self debuglevel 0 printencoding file using iso88591 while datablock data readself blocksize if encode datablock datablock encodeiso88591 sys audithttp client send self datablock self sock sendalldatablock return sys audithttp client send self data try self sock sendalldata except typeerror if isinstancedata collections abc iterable for d in data self sock sendalld else raise typeerrordata should be a byteslike object or an iterable got r typedata def outputself s self buffer appends def readreadableself readable if self debuglevel 0 printreading a readable encode self istextioreadable if encode and self debuglevel 0 printencoding file using iso88591 while datablock readable readself blocksize if encode datablock datablock encodeiso88591 yield datablock def sendoutputself messagebodynone encodechunkedfalse self buffer extendb b msg brn joinself buffer del self buffer self sendmsg if messagebody is not none create a consistent interface to messagebody if hasattrmessagebody read let filelike take precedence over bytelike this is needed to allow the current position of mmap ed files to be taken into account chunks self readreadablemessagebody else try this is solely to check to see if messagebody implements the buffer api it would be easier to capture if pyobjectcheckbuffer was exposed to python memoryviewmessagebody except typeerror try chunks itermessagebody except typeerror raise typeerrormessagebody should be a byteslike object or an iterable got r typemessagebody else the object implements the buffer interface and can be passed directly into socket methods chunks messagebody for chunk in chunks if not chunk if self debuglevel 0 print zero length chunk ignored continue if encodechunked and self httpvsn 11 chunked encoding chunk f lenchunk xrn encode ascii chunk b rn self sendchunk if encodechunked and self httpvsn 11 end chunked transfer self sendb 0rnrn def putrequestself method url skiphostfalse skipacceptencodingfalse if a prior response has been completed then forget about it if self response and self response isclosed self response none in certain cases we cannot issue another request on this connection this occurs when 1 we are in the process of sending a request csreqstarted 2 a response to a previous request has signalled that it is going to close the connection upon completion 3 the headers for the previous response have not been read thus we cannot determine whether point 2 is true csreqsent if there is no prior response then we can request at will if point 2 is true then we will have passed the socket to the response effectively meaning there is no prior response and will open a new one when a new request is made note if a prior response exists then we can start a new request we are not allowed to begin fetching the response to this new request however until that prior response is complete if self state csidle self state csreqstarted else raise cannotsendrequestself state self validatemethodmethod save the method for use later in the response phase self method method url url or self validatepathurl request s s s method url self httpvsnstr self outputself encoderequestrequest if self httpvsn 11 issue some standard headers for better http1 1 compliance if not skiphost this header is issued only for http1 1 connections more specifically this means it is only issued when the client uses the new httpconnection class backwardscompat clients will be using http1 0 and those clients may be issuing this header themselves we should not issue it twice some web servers such as apache barf when they see two host headers if we need a nonstandard port include it in the header if the request is going through a proxy but the host of the actual url not the host of the proxy netloc if url startswith http nil netloc nil nil nil urlspliturl if netloc try netlocenc netloc encodeascii except unicodeencodeerror netlocenc netloc encodeidna self putheader host stripipv6ifacenetlocenc else if self tunnelhost host self tunnelhost port self tunnelport else host self host port self port try hostenc host encodeascii except unicodeencodeerror hostenc host encodeidna as per rfc 273 ipv6 address should be wrapped with when used as host header if in host hostenc b hostenc b hostenc stripipv6ifacehostenc if port self defaultport self putheader host hostenc else hostenc hostenc decodeascii self putheader host s s hostenc port note we are assuming that clients will not attempt to set these headers since this library must deal with the consequences this also means that when the supporting libraries are updated to recognize other forms then this code should be changed removed or updated we only want a contentencoding of identity since we don t support encodings such as xgzip or xdeflate if not skipacceptencoding self putheader acceptencoding identity we can accept chunked transferencodings but no others note no te header implies only chunked self putheader te chunked if te is supplied in the header then it must appear in a connection header self putheader connection te else for http1 0 the server will assume not chunked pass def encoderequestself request ascii also helps prevent cve20199740 return request encode ascii def validatemethodself method prevent http header injection validate a url for putrequest prevent cve20199740 match containsdisallowedurlpcharre searchurl if match raise invalidurlfurl can t contain control characters url r ffound at least match group r def validatehostself host prevent cve201918348 send a request header line to the server for example h putheader accept texthtml indicate that the last header line has been sent to the server this method sends the request to the server the optional messagebody argument can be used to pass a message body associated with the request send a complete request to the server self sendrequestmethod url body headers encodechunked def sendrequestself method url body headers encodechunked honor explicitly requested host and acceptencoding headers headernames frozensetk lower for k in headers skips if host in headernames skips skiphost 1 if acceptencoding in headernames skips skipacceptencoding 1 self putrequestmethod url skips chunked encoding will happen if http1 1 is used and either the caller passes encodechunkedtrue or the following conditions hold 1 contentlength has not been explicitly set 2 the body is a file or iterable but not a str or byteslike 3 transferencoding has not been explicitly set by the caller if contentlength not in headernames only chunk body if not explicitly set for backwards compatibility assuming the client code is already handling the chunking if transferencoding not in headernames if contentlength cannot be automatically determined fall back to chunked encoding encodechunked false contentlength self getcontentlengthbody method if contentlength is none if body is not none if self debuglevel 0 print unable to determine size of r body encodechunked true self putheader transferencoding chunked else self putheader contentlength strcontentlength else encodechunked false for hdr value in headers items self putheaderhdr value if isinstancebody str rfc 2616 section 3 7 1 says that text default has a default charset of iso88591 body encodebody body self endheadersbody encodechunkedencodechunked def getresponseself if a prior response has been completed then forget about it if self response and self response isclosed self response none if a prior response exists then it must be completed otherwise we cannot read this response s header to determine the connectionclose behavior note if a prior response existed but was connectionclose then the socket and response were made independent of this httpconnection object since a new request requires that we open a whole new connection this means the prior response had one of two states 1 willclose this connection was reset and the prior socket and response operate independently 2 persistent the response was retained and we await its isclosed status to become true if self state csreqsent or self response raise responsenotreadyself state if self debuglevel 0 response self responseclassself sock self debuglevel methodself method else response self responseclassself sock methodself method try try response begin except connectionerror self close raise assert response willclose unknown self state csidle if response willclose this effectively passes the connection to the response self close else remember this so we can tell when it is complete self response response return response except response close raise try import ssl except importerror pass else class httpsconnectionhttpconnection this class allows communication via ssl defaultport httpsport def initself host portnone timeoutsocket globaldefaulttimeout sourceaddressnone contextnone blocksize8192 superhttpsconnection self inithost port timeout sourceaddress blocksizeblocksize if context is none context createhttpscontextself httpvsn self context context def connectself connect to a host on a given ssl port super connect if self tunnelhost serverhostname self tunnelhost else serverhostname self host self sock self context wrapsocketself sock serverhostnameserverhostname all appendhttpsconnection class httpexceptionexception subclasses that define an init must call exception init or define self args otherwise str will fail pass class notconnectedhttpexception pass class invalidurlhttpexception pass class unknownprotocolhttpexception def initself version self args version self version version class unknowntransferencodinghttpexception pass class unimplementedfilemodehttpexception pass class incompletereadhttpexception def initself partial expectednone self args partial self partial partial self expected expected def reprself if self expected is not none e i more expected self expected else e return si bytes reads self class name lenself partial e str object str class improperconnectionstatehttpexception pass class cannotsendrequestimproperconnectionstate pass class cannotsendheaderimproperconnectionstate pass class responsenotreadyimproperconnectionstate pass class badstatuslinehttpexception def initself line if not line line reprline self args line self line line class linetoolonghttpexception def initself linetype httpexception initself got more than d bytes when reading s maxline linetype class remotedisconnectedconnectionreseterror badstatusline def initself pos kw badstatusline initself connectionreseterror initself pos kw for backwards compatibility error httpexception http 1 1 client library intro stuff goes here other stuff too httpconnection goes through a number of states which define when a client may legally make another request or fetch the response for a particular request this diagram details these state transitions null httpconnection v idle putrequest v request started putheader endheaders v request sent _____________________________ getresponse raises response getresponse connectionerror v v unread response idle response headers read ____________________ response read putrequest v v idle req started unread response ______ response read putheader endheaders v v request started req sent unread response response read v request sent this diagram presents the following rules a second request may not be started until response headers read a response object cannot be retrieved until request sent there is no differentiation between an unread response body and a partially read response body note this enforcement is applied by the httpconnection class the httpresponse class does not enforce this state machine which implies sophisticated clients may accelerate the request response pipeline caution should be taken though accelerating the states beyond the above pattern may imply knowledge of the server s connection close behavior for certain requests for example it is impossible to tell whether the server will close the connection until the response headers have been read this means that further requests cannot be placed into the pipeline until it is known that the server will not be closing the connection logical state __state __response idle _cs_idle none request started _cs_req_started none request sent _cs_req_sent none unread response _cs_idle response_class req started unread response _cs_req_started response_class req sent unread response _cs_req_sent response_class httpmessage parse_headers and the http status code constants are intentionally omitted for simplicity connection states hack to maintain backwards compatibility another hack to maintain backwards compatibility mapping status codes to official w3c names maximal line length when calling readline header name value abnf http tools ietf org html rfc7230 section 3 2 vchar x21 7e obs text x80 ff header field field name ows field value ows field name token field value field content obs fold field content field vchar 1 sp htab field vchar field vchar vchar obs text obs fold crlf 1 sp htab obsolete line folding see section 3 2 4 token 1 tchar tchar _ digit alpha any vchar except delimiters vchar defined in http tools ietf org html rfc5234 appendix b 1 the patterns for both name and value are more lenient than rfc definitions to allow for backwards compatibility these characters are not allowed within http url paths see https tools ietf org html rfc3986 section 3 3 and the https tools ietf org html rfc3986 appendix a pchar definition prevents cve 2019 9740 includes control characters such as r n we don t restrict chars above x7f as putrequest limits us to ascii arguably only these _should_ allowed _is_allowed_url_pchars_re re compile r a za z0 9 _ we are more lenient for assumed real world compatibility purposes these characters are not allowed within http method names to prevent http header injection we always set the content length header for these methods because some servers will otherwise respond with a 411 call data encode latin 1 but show a better error message remove interface scope from ipv6 address xxx the only usage of this method is in http server cgihttprequesthandler maybe move the code there so that it doesn t need to be part of the public api the api has never been defined so this could cause backwards compatibility issues find all header lines matching a given header name look through the list of headers and find all lines matching a given header name and their continuation lines a list of the lines is returned without interpretation if the header does not occur an empty list is returned if the header occurs multiple times all occurrences are returned case is not important in the header name reads potential header lines into a list from a file pointer length of line is limited by _maxline and number of headers is limited by _maxheaders parses only rfc2822 headers from header lines email parser wants to see strings rather than bytes but a textiowrapper around self rfile would buffer too many bytes from the stream bytes which we later need to read as bytes so we read the correct bytes here as bytes for email parser to parse parses only rfc2822 headers from a file pointer see rfc 2616 sec 19 6 and rfc 1945 sec 6 for details the bytes from the socket object are iso 8859 1 strings see rfc 2616 sec 2 2 which notes an exception for mime encoded text following rfc 2047 the basic status line parsing only accepts iso 8859 1 if the response includes a content length header we need to make sure that the client doesn t read more than the specified number of bytes if it does it will block until the server times out and closes the connection this will happen if a self fp read is done without a size whether self fp is buffered or not so no self fp read by clients unless they know what they are doing the httpresponse object is returned via urllib the clients of http and urllib expect different attributes for the headers headers is used here and supports urllib msg is provided as a backwards compatibility layer for http clients from the status line of the response http version status code reason phrase is chunked being used bytes left to read in current chunk number of bytes left in response conn will close at end of response presumably the server closed the connection before sending a valid response empty version will cause next test to fail the status code is a three digit number we ve already started reading the response read until we get a non 100 response skip the header from the 100 response some servers might still return 0 9 treat it as 1 0 anyway use http 1 1 code for http 1 x where x 1 are we using the chunked style of transfer encoding will the connection close at the end of the response do we have a content length note rfc 2616 s4 4 3 says we ignore this if tr_enc is chunked ignore nonsensical negative lengths does the body have a fixed length of zero 1xx codes if the connection remains open and we aren t using chunked and a content length was not provided then assume that the connection will close an http 1 1 proxy is assumed to stay open unless explicitly closed some http 1 0 implementations have support for persistent connections using rules different than http 1 1 for older http keep alive indicates persistent connection at least akamai returns a connection keep alive header which was supposed to be sent by the client proxy connection is a netscape hack otherwise assume it will close set closed flag these implementations are for the benefit of io bufferedreader xxx this class should probably be revised to act more like the raw stream that bufferedreader expects always returns true end of raw stream methods true if the connection is closed note it is possible that we will not ever call self close this case occurs when will_close is true length is none and we read up to the last byte but not past it implies if will_close is false then self close will always be called meaning self isclosed is meaningful read and return the response body or up to the next amt bytes clip the read to the end of response ideally we would raise incompleteread if the content length wasn t satisfied but it might break compatibility amount is not given unbounded read so we must check self length we read everything read up to len b bytes into bytearray b and return the number of bytes read clip the read to the end of response we do not use _safe_read here because this may be a will_close connection and the user is reading more bytes than will be provided for example reading in 1k chunks ideally we would raise incompleteread if the content length wasn t satisfied but it might break compatibility read the next chunk size from the file strip chunk extensions close the connection as protocol synchronisation is probably lost read and discard trailer up to the crlf terminator note we shouldn t have any trailers a vanishingly small number of sites eof without sending the trailer return self chunk_left reading a new chunk if necessary chunk_left 0 at the end of the current chunk need to close it chunk_left none no current chunk should read next this function returns non zero or none if the last chunk has been read can be 0 or none we are at the end of chunk discard chunk end toss the crlf at the end of the chunk last chunk 1 0 chunk extension crlf we read everything close the file read the number of bytes requested this function should be used when amt bytes should be present for reading if the bytes are truly not available due to eof then the incompleteread exception can be used to detect the problem same as _safe_read but for reading into a buffer read with at most one underlying system call if at least one byte is buffered return that instead having this enables iobase readline to read more than one byte at a time fallback to iobase readline which uses peek and read strictly speaking _get_chunk_left may cause more than one read but that is ok since that is to satisfy the chunked protocol if n is negative or larger than chunk_left strictly speaking _get_chunk_left may cause more than one read but that is ok since that is to satisfy the chunked protocol peek doesn t worry about protocol eof peek is allowed to return more than requested just request the entire chunk and truncate what we get returns the value of the header matching name if there are multiple matching headers the values are combined into a single string separated by commas and spaces if no matching header is found returns default or none if the default is not specified if the headers are unknown raises http client responsenotready return list of header value tuples we override iobase __iter__ so that it doesn t check for closed ness for compatibility with old style urllib responses returns an instance of the class mimetools message containing meta information associated with the url when the method is http these headers are those returned by the server at the head of the retrieved html page including content length and content type when the method is ftp a content length header will be present if as is now usual the server passed back a file length in response to the ftp retrieval request a content type header will be present if the mime type can be guessed when the method is local file returned headers will include a date representing the file s last modified time a content length giving file size and a content type containing a guess at the file s type see also the description of the mimetools module return the real url of the page in some cases the http server redirects a client to another url the urlopen function handles this transparently but in some cases the caller needs to know which url the client was redirected to the geturl method can be used to get at this redirected url return the http status code that was sent with the response or none if the url is not an http url function also used by urllib request to be able to set the check_hostname attribute on a context object send alpn extension to indicate http 1 1 protocol enable pha for tls 1 3 connections if available test whether a file like object is a text or a binary stream get the content length based on the body if the body is none we set content length 0 for methods that expect a body rfc 7230 section 3 3 2 we also set the content length for any method if the body is a str or bytes like object and not a file do an explicit check for not none here to distinguish between unset and set but empty file like object does it implement the buffer protocol bytes bytearray array this is stored as an instance variable to allow unit tests to replace it with a suitable mockup set up host and port for http connect tunnelling in a connection that uses http connect tunnelling the host passed to the constructor is used as a proxy server that relays all communication to the endpoint passed to set_tunnel this done by sending an http connect request to the proxy server when the connection is established this method must be called before the http connection has been established the headers argument should be a mapping of extra http headers to send with the connect request as http 1 1 is used for http connect tunnelling request as per the rfc https tools ietf org html rfc7231 section 4 3 6 a http host header must be provided matching the ity form of the request target provided as the destination for the connect request if a http host header is not provided via the headers argument one is generated and transmitted automatically ipv6 addresses have http foo com http foo com making a single send call instead of one per line encourages the host os to use a more optimal packet size instead of potentially emitting a series of small packets returns a dictionary with the headers of the response received from the proxy server to the connect request sent to set the tunnel if the connect request was not sent the method returns none connect to the host and port specified in __init__ might fail in oss that don t implement tcp_nodelay close the connection to the http server close it manually there may be other refs send data to the server data can be a string object a bytes object an array object a file like object that supports a read method or an iterable object add a line of output to the current request buffer assumes that the line does not end with r n send the currently buffered request and clear the buffer appends an extra r n to the buffer a message_body may be specified to be appended to the request create a consistent interface to message_body let file like take precedence over byte like this is needed to allow the current position of mmap ed files to be taken into account this is solely to check to see if message_body implements the buffer api it would be easier to capture if pyobject_checkbuffer was exposed to python the object implements the buffer interface and can be passed directly into socket methods chunked encoding end chunked transfer send a request to the server method specifies an http request method e g get url specifies the object being requested e g index html skip_host if true does not add automatically a host header skip_accept_encoding if true does not add automatically an accept encoding header if a prior response has been completed then forget about it in certain cases we cannot issue another request on this connection this occurs when 1 we are in the process of sending a request _cs_req_started 2 a response to a previous request has signalled that it is going to close the connection upon completion 3 the headers for the previous response have not been read thus we cannot determine whether point 2 is true _cs_req_sent if there is no prior response then we can request at will if point 2 is true then we will have passed the socket to the response effectively meaning there is no prior response and will open a new one when a new request is made note if a prior response exists then we can start a new request we are not allowed to begin fetching the response to this new request however until that prior response is complete save the method for use later in the response phase issue some standard headers for better http 1 1 compliance this header is issued only for http 1 1 connections more specifically this means it is only issued when the client uses the new httpconnection class backwards compat clients will be using http 1 0 and those clients may be issuing this header themselves we should not issue it twice some web servers such as apache barf when they see two host headers if we need a non standard port include it in the header if the request is going through a proxy but the host of the actual url not the host of the proxy as per rfc 273 ipv6 address should be wrapped with when used as host header note we are assuming that clients will not attempt to set these headers since this library must deal with the consequences this also means that when the supporting libraries are updated to recognize other forms then this code should be changed removed or updated we only want a content encoding of identity since we don t support encodings such as x gzip or x deflate we can accept chunked transfer encodings but no others note no te header implies only chunked self putheader te chunked if te is supplied in the header then it must appear in a connection header self putheader connection te for http 1 0 the server will assume not chunked ascii also helps prevent cve 2019 9740 validate a method name for putrequest prevent http header injection validate a url for putrequest prevent cve 2019 9740 validate a host so it doesn t contain control characters prevent cve 2019 18348 send a request header line to the server for example h putheader accept text html indicate that the last header line has been sent to the server this method sends the request to the server the optional message_body argument can be used to pass a message body associated with the request send a complete request to the server honor explicitly requested host and accept encoding headers chunked encoding will happen if http 1 1 is used and either the caller passes encode_chunked true or the following conditions hold 1 content length has not been explicitly set 2 the body is a file or iterable but not a str or bytes like 3 transfer encoding has not been explicitly set by the caller only chunk body if not explicitly set for backwards compatibility assuming the client code is already handling the chunking if content length cannot be automatically determined fall back to chunked encoding rfc 2616 section 3 7 1 says that text default has a default charset of iso 8859 1 get the response from the server if the httpconnection is in the correct state returns an instance of httpresponse or of whatever object is returned by the response_class variable if a request has not been sent or if a previous response has not be handled responsenotready is raised if the http response indicates that the connection should be closed then it will be closed before the response is returned when the connection is closed the underlying socket is closed if a prior response has been completed then forget about it if a prior response exists then it must be completed otherwise we cannot read this response s header to determine the connection close behavior note if a prior response existed but was connection close then the socket and response were made independent of this httpconnection object since a new request requires that we open a whole new connection this means the prior response had one of two states 1 will_close this connection was reset and the prior socket and response operate independently 2 persistent the response was retained and we await its isclosed status to become true this effectively passes the connection to the response remember this so we can tell when it is complete subclasses that define an __init__ must call exception __init__ or define self args otherwise str will fail for backwards compatibility
r

import email.parser
import email.message
import errno
import http
import io
import re
import socket
import sys
import collections.abc
from urllib.parse import urlsplit



__all__ = ["HTTPResponse", "HTTPConnection",
           "HTTPException", "NotConnected", "UnknownProtocol",
           "UnknownTransferEncoding", "UnimplementedFileMode",
           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
           "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error",
           "responses"]

HTTP_PORT = 80
HTTPS_PORT = 443

_UNKNOWN = 'UNKNOWN'


_CS_IDLE = 'Idle'
_CS_REQ_STARTED = 'Request-started'
_CS_REQ_SENT = 'Request-sent'



globals().update(http.HTTPStatus.__members__)



responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()}


_MAXLINE = 65536
_MAXHEADERS = 100

_is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch
_is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search

_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')

_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]')


_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}


def _encode(data, name='data'):
    
    try:
        return data.encode("latin-1")
    except UnicodeEncodeError as err:
        raise UnicodeEncodeError(
            err.encoding,
            err.object,
            err.start,
            err.end,
            "%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') "
            "if you want to send it encoded in UTF-8." %
            (name.title(), data[err.start:err.end], name)) from None

def _strip_ipv6_iface(enc_name: bytes) -> bytes:
    
    enc_name, percent, _ = enc_name.partition(b"%")
    if percent:
        assert enc_name.startswith(b'['), enc_name
        enc_name += b']'
    return enc_name

class HTTPMessage(email.message.Message):
    
    def getallmatchingheaders(self, name):
        
        name = name.lower() + ':'
        n = len(name)
        lst = []
        hit = 0
        for line in self.keys():
            if line[:n].lower() == name:
                hit = 1
            elif not line[:1].isspace():
                hit = 0
            if hit:
                lst.append(line)
        return lst

def _read_headers(fp):
    
    headers = []
    while True:
        line = fp.readline(_MAXLINE + 1)
        if len(line) > _MAXLINE:
            raise LineTooLong("header line")
        headers.append(line)
        if len(headers) > _MAXHEADERS:
            raise HTTPException("got more than %d headers" % _MAXHEADERS)
        if line in (b'\r\n', b'\n', b''):
            break
    return headers

def _parse_header_lines(header_lines, _class=HTTPMessage):
    
    hstring = b''.join(header_lines).decode('iso-8859-1')
    return email.parser.Parser(_class=_class).parsestr(hstring)

def parse_headers(fp, _class=HTTPMessage):
    

    headers = _read_headers(fp)
    return _parse_header_lines(headers, _class)


class HTTPResponse(io.BufferedIOBase):

    
    def __init__(self, sock, debuglevel=0, method=None, url=None):
        
        self.fp = sock.makefile("rb")
        self.debuglevel = debuglevel
        self._method = method
        self.headers = self.msg = None

        self.version = _UNKNOWN 
        self.status = _UNKNOWN  
        self.reason = _UNKNOWN  

        self.chunked = _UNKNOWN         
        self.chunk_left = _UNKNOWN      
        self.length = _UNKNOWN          
        self.will_close = _UNKNOWN      

    def _read_status(self):
        line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
        if len(line) > _MAXLINE:
            raise LineTooLong("status line")
        if self.debuglevel > 0:
            print("reply:", repr(line))
        if not line:
            
            
            raise RemoteDisconnected("Remote end closed connection without"
                                     " response")
        try:
            version, status, reason = line.split(None, 2)
        except ValueError:
            try:
                version, status = line.split(None, 1)
                reason = ""
            except ValueError:
                
                version = ""
        if not version.startswith("HTTP/"):
            self._close_conn()
            raise BadStatusLine(line)

        
        try:
            status = int(status)
            if status < 100 or status > 999:
                raise BadStatusLine(line)
        except ValueError:
            raise BadStatusLine(line)
        return version, status, reason

    def begin(self):
        if self.headers is not None:
            
            return

        
        while True:
            version, status, reason = self._read_status()
            if status != CONTINUE:
                break
            
            skipped_headers = _read_headers(self.fp)
            if self.debuglevel > 0:
                print("headers:", skipped_headers)
            del skipped_headers

        self.code = self.status = status
        self.reason = reason.strip()
        if version in ("HTTP/1.0", "HTTP/0.9"):
            
            self.version = 10
        elif version.startswith("HTTP/1."):
            self.version = 11   
        else:
            raise UnknownProtocol(version)

        self.headers = self.msg = parse_headers(self.fp)

        if self.debuglevel > 0:
            for hdr, val in self.headers.items():
                print("header:", hdr + ":", val)

        
        tr_enc = self.headers.get("transfer-encoding")
        if tr_enc and tr_enc.lower() == "chunked":
            self.chunked = True
            self.chunk_left = None
        else:
            self.chunked = False

        
        self.will_close = self._check_close()

        
        
        self.length = None
        length = self.headers.get("content-length")
        if length and not self.chunked:
            try:
                self.length = int(length)
            except ValueError:
                self.length = None
            else:
                if self.length < 0:  
                    self.length = None
        else:
            self.length = None

        
        if (status == NO_CONTENT or status == NOT_MODIFIED or
            100 <= status < 200 or      
            self._method == "HEAD"):
            self.length = 0

        
        if (not self.will_close and
            not self.chunked and
            self.length is None):
            self.will_close = True

    def _check_close(self):
        conn = self.headers.get("connection")
        if self.version == 11:
            
            
            if conn and "close" in conn.lower():
                return True
            return False

        if self.headers.get("keep-alive"):
            return False

        if conn and "keep-alive" in conn.lower():
            return False

        
        pconn = self.headers.get("proxy-connection")
        if pconn and "keep-alive" in pconn.lower():
            return False

        
        return True

    def _close_conn(self):
        fp = self.fp
        self.fp = None
        fp.close()

    def close(self):
        try:
            super().close() 
        finally:
            if self.fp:
                self._close_conn()


    def flush(self):
        super().flush()
        if self.fp:
            self.fp.flush()

    def readable(self):
        
        return True

    
    def isclosed(self):
        
        return self.fp is None

    def read(self, amt=None):
        
        if self.fp is None:
            return b""

        if self._method == "HEAD":
            self._close_conn()
            return b""

        if self.chunked:
            return self._read_chunked(amt)

        if amt is not None:
            if self.length is not None and amt > self.length:
                
                amt = self.length
            s = self.fp.read(amt)
            if not s and amt:
                
                
                self._close_conn()
            elif self.length is not None:
                self.length -= len(s)
                if not self.length:
                    self._close_conn()
            return s
        else:
            
            if self.length is None:
                s = self.fp.read()
            else:
                try:
                    s = self._safe_read(self.length)
                except IncompleteRead:
                    self._close_conn()
                    raise
                self.length = 0
            self._close_conn()        
            return s

    def readinto(self, b):
        

        if self.fp is None:
            return 0

        if self._method == "HEAD":
            self._close_conn()
            return 0

        if self.chunked:
            return self._readinto_chunked(b)

        if self.length is not None:
            if len(b) > self.length:
                
                b = memoryview(b)[0:self.length]


        n = self.fp.readinto(b)
        if not n and b:
            
            
            self._close_conn()
        elif self.length is not None:
            self.length -= n
            if not self.length:
                self._close_conn()
        return n

    def _read_next_chunk_size(self):
        
        line = self.fp.readline(_MAXLINE + 1)
        if len(line) > _MAXLINE:
            raise LineTooLong("chunk size")
        i = line.find(b";")
        if i >= 0:
            line = line[:i] 
        try:
            return int(line, 16)
        except ValueError:
            
            
            self._close_conn()
            raise

    def _read_and_discard_trailer(self):
        
        
        while True:
            line = self.fp.readline(_MAXLINE + 1)
            if len(line) > _MAXLINE:
                raise LineTooLong("trailer line")
            if not line:
                
                
                break
            if line in (b'\r\n', b'\n', b''):
                break

    def _get_chunk_left(self):
        
        chunk_left = self.chunk_left
        if not chunk_left: 
            if chunk_left is not None:
                
                self._safe_read(2)  
            try:
                chunk_left = self._read_next_chunk_size()
            except ValueError:
                raise IncompleteRead(b'')
            if chunk_left == 0:
                
                self._read_and_discard_trailer()
                
                self._close_conn()
                chunk_left = None
            self.chunk_left = chunk_left
        return chunk_left

    def _read_chunked(self, amt=None):
        assert self.chunked != _UNKNOWN
        value = []
        try:
            while (chunk_left := self._get_chunk_left()) is not None:
                if amt is not None and amt <= chunk_left:
                    value.append(self._safe_read(amt))
                    self.chunk_left = chunk_left - amt
                    break

                value.append(self._safe_read(chunk_left))
                if amt is not None:
                    amt -= chunk_left
                self.chunk_left = 0
            return b''.join(value)
        except IncompleteRead as exc:
            raise IncompleteRead(b''.join(value)) from exc

    def _readinto_chunked(self, b):
        assert self.chunked != _UNKNOWN
        total_bytes = 0
        mvb = memoryview(b)
        try:
            while True:
                chunk_left = self._get_chunk_left()
                if chunk_left is None:
                    return total_bytes

                if len(mvb) <= chunk_left:
                    n = self._safe_readinto(mvb)
                    self.chunk_left = chunk_left - n
                    return total_bytes + n

                temp_mvb = mvb[:chunk_left]
                n = self._safe_readinto(temp_mvb)
                mvb = mvb[n:]
                total_bytes += n
                self.chunk_left = 0

        except IncompleteRead:
            raise IncompleteRead(bytes(b[0:total_bytes]))

    def _safe_read(self, amt):
        
        data = self.fp.read(amt)
        if len(data) < amt:
            raise IncompleteRead(data, amt-len(data))
        return data

    def _safe_readinto(self, b):
        
        amt = len(b)
        n = self.fp.readinto(b)
        if n < amt:
            raise IncompleteRead(bytes(b[:n]), amt-n)
        return n

    def read1(self, n=-1):
        
        if self.fp is None or self._method == "HEAD":
            return b""
        if self.chunked:
            return self._read1_chunked(n)
        if self.length is not None and (n < 0 or n > self.length):
            n = self.length
        result = self.fp.read1(n)
        if not result and n:
            self._close_conn()
        elif self.length is not None:
            self.length -= len(result)
        return result

    def peek(self, n=-1):
        
        
        if self.fp is None or self._method == "HEAD":
            return b""
        if self.chunked:
            return self._peek_chunked(n)
        return self.fp.peek(n)

    def readline(self, limit=-1):
        if self.fp is None or self._method == "HEAD":
            return b""
        if self.chunked:
            
            return super().readline(limit)
        if self.length is not None and (limit < 0 or limit > self.length):
            limit = self.length
        result = self.fp.readline(limit)
        if not result and limit:
            self._close_conn()
        elif self.length is not None:
            self.length -= len(result)
        return result

    def _read1_chunked(self, n):
        
        
        chunk_left = self._get_chunk_left()
        if chunk_left is None or n == 0:
            return b''
        if not (0 <= n <= chunk_left):
            n = chunk_left 
        read = self.fp.read1(n)
        self.chunk_left -= len(read)
        if not read:
            raise IncompleteRead(b"")
        return read

    def _peek_chunked(self, n):
        
        
        try:
            chunk_left = self._get_chunk_left()
        except IncompleteRead:
            return b'' 
        if chunk_left is None:
            return b'' 
        
        
        return self.fp.peek(chunk_left)[:chunk_left]

    def fileno(self):
        return self.fp.fileno()

    def getheader(self, name, default=None):
        
        if self.headers is None:
            raise ResponseNotReady()
        headers = self.headers.get_all(name) or default
        if isinstance(headers, str) or not hasattr(headers, '__iter__'):
            return headers
        else:
            return ', '.join(headers)

    def getheaders(self):
        
        if self.headers is None:
            raise ResponseNotReady()
        return list(self.headers.items())
    

    def __iter__(self):
        return self

    

    def info(self):
        
        return self.headers

    def geturl(self):
        
        return self.url

    def getcode(self):
        
        return self.status


def _create_https_context(http_version):
    
    
    context = ssl._create_default_https_context()
    
    if http_version == 11:
        context.set_alpn_protocols(['http/1.1'])
    
    if context.post_handshake_auth is not None:
        context.post_handshake_auth = True
    return context


class HTTPConnection:

    _http_vsn = 11
    _http_vsn_str = 'HTTP/1.1'

    response_class = HTTPResponse
    default_port = HTTP_PORT
    auto_open = 1
    debuglevel = 0

    @staticmethod
    def _is_textIO(stream):
        
        return isinstance(stream, io.TextIOBase)

    @staticmethod
    def _get_content_length(body, method):
        
        if body is None:
            
        
            if method.upper() in _METHODS_EXPECTING_BODY:
                return 0
            else:
                return None

        if hasattr(body, 'read'):
            
            return None

        try:
            
            mv = memoryview(body)
            return mv.nbytes
        except TypeError:
            pass

        if isinstance(body, str):
            return len(body)

        return None

    def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                 source_address=None, blocksize=8192):
        self.timeout = timeout
        self.source_address = source_address
        self.blocksize = blocksize
        self.sock = None
        self._buffer = []
        self.__response = None
        self.__state = _CS_IDLE
        self._method = None
        self._tunnel_host = None
        self._tunnel_port = None
        self._tunnel_headers = {}
        self._raw_proxy_headers = None

        (self.host, self.port) = self._get_hostport(host, port)

        self._validate_host(self.host)

        
        
        self._create_connection = socket.create_connection

    def set_tunnel(self, host, port=None, headers=None):
        

        if self.sock:
            raise RuntimeError("Can't set up tunnel for established connection")

        self._tunnel_host, self._tunnel_port = self._get_hostport(host, port)
        if headers:
            self._tunnel_headers = headers.copy()
        else:
            self._tunnel_headers.clear()

        if not any(header.lower() == "host" for header in self._tunnel_headers):
            encoded_host = self._tunnel_host.encode("idna").decode("ascii")
            self._tunnel_headers["Host"] = "%s:%d" % (
                encoded_host, self._tunnel_port)

    def _get_hostport(self, host, port):
        if port is None:
            i = host.rfind(':')
            j = host.rfind(']')         
            if i > j:
                try:
                    port = int(host[i+1:])
                except ValueError:
                    if host[i+1:] == "": 
                        port = self.default_port
                    else:
                        raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
                host = host[:i]
            else:
                port = self.default_port
            if host and host[0] == '[' and host[-1] == ']':
                host = host[1:-1]

        return (host, port)

    def set_debuglevel(self, level):
        self.debuglevel = level

    def _tunnel(self):
        connect = b"CONNECT %s:%d %s\r\n" % (
            self._tunnel_host.encode("idna"), self._tunnel_port,
            self._http_vsn_str.encode("ascii"))
        headers = [connect]
        for header, value in self._tunnel_headers.items():
            headers.append(f"{header}: {value}\r\n".encode("latin-1"))
        headers.append(b"\r\n")
        
        self.send(b"".join(headers))
        del headers

        response = self.response_class(self.sock, method=self._method)
        try:
            (version, code, message) = response._read_status()

            self._raw_proxy_headers = _read_headers(response.fp)

            if self.debuglevel > 0:
                for header in self._raw_proxy_headers:
                    print('header:', header.decode())

            if code != http.HTTPStatus.OK:
                self.close()
                raise OSError(f"Tunnel connection failed: {code} {message.strip()}")

        finally:
            response.close()

    def get_proxy_response_headers(self):
        
        return (
            _parse_header_lines(self._raw_proxy_headers)
            if self._raw_proxy_headers is not None
            else None
        )

    def connect(self):
        
        sys.audit("http.client.connect", self, self.host, self.port)
        self.sock = self._create_connection(
            (self.host,self.port), self.timeout, self.source_address)
        
        try:
            self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        except OSError as e:
            if e.errno != errno.ENOPROTOOPT:
                raise

        if self._tunnel_host:
            self._tunnel()

    def close(self):
        
        self.__state = _CS_IDLE
        try:
            sock = self.sock
            if sock:
                self.sock = None
                sock.close()   
        finally:
            response = self.__response
            if response:
                self.__response = None
                response.close()

    def send(self, data):
        

        if self.sock is None:
            if self.auto_open:
                self.connect()
            else:
                raise NotConnected()

        if self.debuglevel > 0:
            print("send:", repr(data))
        if hasattr(data, "read") :
            if self.debuglevel > 0:
                print("sending a readable")
            encode = self._is_textIO(data)
            if encode and self.debuglevel > 0:
                print("encoding file using iso-8859-1")
            while datablock := data.read(self.blocksize):
                if encode:
                    datablock = datablock.encode("iso-8859-1")
                sys.audit("http.client.send", self, datablock)
                self.sock.sendall(datablock)
            return
        sys.audit("http.client.send", self, data)
        try:
            self.sock.sendall(data)
        except TypeError:
            if isinstance(data, collections.abc.Iterable):
                for d in data:
                    self.sock.sendall(d)
            else:
                raise TypeError("data should be a bytes-like object "
                                "or an iterable, got %r" % type(data))

    def _output(self, s):
        
        self._buffer.append(s)

    def _read_readable(self, readable):
        if self.debuglevel > 0:
            print("reading a readable")
        encode = self._is_textIO(readable)
        if encode and self.debuglevel > 0:
            print("encoding file using iso-8859-1")
        while datablock := readable.read(self.blocksize):
            if encode:
                datablock = datablock.encode("iso-8859-1")
            yield datablock

    def _send_output(self, message_body=None, encode_chunked=False):
        
        self._buffer.extend((b"", b""))
        msg = b"\r\n".join(self._buffer)
        del self._buffer[:]
        self.send(msg)

        if message_body is not None:

            if hasattr(message_body, 'read'):
                
                chunks = self._read_readable(message_body)

            else:

                try:
                    memoryview(message_body)
                except TypeError:
                    try:
                        chunks = iter(message_body)
                    except TypeError:
                        raise TypeError("message_body should be a bytes-like "
                                        "object or an iterable, got %r"
                                        % type(message_body))
                else:
                    
                    
                    chunks = (message_body,)

            for chunk in chunks:
                if not chunk:
                    if self.debuglevel > 0:
                        print('Zero length chunk ignored')
                    continue

                if encode_chunked and self._http_vsn == 11:
                    
                    chunk = f'{len(chunk):X}\r\n'.encode('ascii') + chunk \
                        + b'\r\n'
                self.send(chunk)

            if encode_chunked and self._http_vsn == 11:
                
                self.send(b'0\r\n\r\n')

    def putrequest(self, method, url, skip_host=False,
                   skip_accept_encoding=False):
        
        if self.__response and self.__response.isclosed():
            self.__response = None
        
        if self.__state == _CS_IDLE:
            self.__state = _CS_REQ_STARTED
        else:
            raise CannotSendRequest(self.__state)

        self._validate_method(method)

        
        self._method = method

        url = url or '/'
        self._validate_path(url)

        request = '%s %s %s' % (method, url, self._http_vsn_str)

        self._output(self._encode_request(request))

        if self._http_vsn == 11:
            

            if not skip_host:
                
                netloc = ''
                if url.startswith('http'):
                    nil, netloc, nil, nil, nil = urlsplit(url)

                if netloc:
                    try:
                        netloc_enc = netloc.encode("ascii")
                    except UnicodeEncodeError:
                        netloc_enc = netloc.encode("idna")
                    self.putheader('Host', _strip_ipv6_iface(netloc_enc))
                else:
                    if self._tunnel_host:
                        host = self._tunnel_host
                        port = self._tunnel_port
                    else:
                        host = self.host
                        port = self.port

                    try:
                        host_enc = host.encode("ascii")
                    except UnicodeEncodeError:
                        host_enc = host.encode("idna")

                    if ":" in host:
                        host_enc = b'[' + host_enc + b']'
                        host_enc = _strip_ipv6_iface(host_enc)

                    if port == self.default_port:
                        self.putheader('Host', host_enc)
                    else:
                        host_enc = host_enc.decode("ascii")
                        self.putheader('Host', "%s:%s" % (host_enc, port))

            
            if not skip_accept_encoding:
                self.putheader('Accept-Encoding', 'identity')

        else:
            
            pass

    def _encode_request(self, request):
        
        return request.encode('ascii')

    def _validate_method(self, method):
        
        
        match = _contains_disallowed_method_pchar_re.search(method)
        if match:
            raise ValueError(
                    f"method can't contain control characters. {method!r} "
                    f"(found at least {match.group()!r})")

    def _validate_path(self, url):
        
        
        match = _contains_disallowed_url_pchar_re.search(url)
        if match:
            raise InvalidURL(f"URL can't contain control characters. {url!r} "
                             f"(found at least {match.group()!r})")

    def _validate_host(self, host):
        
        
        match = _contains_disallowed_url_pchar_re.search(host)
        if match:
            raise InvalidURL(f"URL can't contain control characters. {host!r} "
                             f"(found at least {match.group()!r})")

    def putheader(self, header, *values):
        
        if self.__state != _CS_REQ_STARTED:
            raise CannotSendHeader()

        if hasattr(header, 'encode'):
            header = header.encode('ascii')

        if not _is_legal_header_name(header):
            raise ValueError('Invalid header name %r' % (header,))

        values = list(values)
        for i, one_value in enumerate(values):
            if hasattr(one_value, 'encode'):
                values[i] = one_value.encode('latin-1')
            elif isinstance(one_value, int):
                values[i] = str(one_value).encode('ascii')

            if _is_illegal_header_value(values[i]):
                raise ValueError('Invalid header value %r' % (values[i],))

        value = b'\r\n\t'.join(values)
        header = header + b': ' + value
        self._output(header)

    def endheaders(self, message_body=None, *, encode_chunked=False):
        
        if self.__state == _CS_REQ_STARTED:
            self.__state = _CS_REQ_SENT
        else:
            raise CannotSendHeader()
        self._send_output(message_body, encode_chunked=encode_chunked)

    def request(self, method, url, body=None, headers={}, *,
                encode_chunked=False):
        
        self._send_request(method, url, body, headers, encode_chunked)

    def _send_request(self, method, url, body, headers, encode_chunked):
        
        header_names = frozenset(k.lower() for k in headers)
        skips = {}
        if 'host' in header_names:
            skips['skip_host'] = 1
        if 'accept-encoding' in header_names:
            skips['skip_accept_encoding'] = 1

        self.putrequest(method, url, **skips)

        if 'content-length' not in header_names:
            
            if 'transfer-encoding' not in header_names:
                
                
                encode_chunked = False
                content_length = self._get_content_length(body, method)
                if content_length is None:
                    if body is not None:
                        if self.debuglevel > 0:
                            print('Unable to determine size of %r' % body)
                        encode_chunked = True
                        self.putheader('Transfer-Encoding', 'chunked')
                else:
                    self.putheader('Content-Length', str(content_length))
        else:
            encode_chunked = False

        for hdr, value in headers.items():
            self.putheader(hdr, value)
        if isinstance(body, str):
            
            
            body = _encode(body, 'body')
        self.endheaders(body, encode_chunked=encode_chunked)

    def getresponse(self):
        
        if self.__response and self.__response.isclosed():
            self.__response = None
        
        if self.__state != _CS_REQ_SENT or self.__response:
            raise ResponseNotReady(self.__state)

        if self.debuglevel > 0:
            response = self.response_class(self.sock, self.debuglevel,
                                           method=self._method)
        else:
            response = self.response_class(self.sock, method=self._method)

        try:
            try:
                response.begin()
            except ConnectionError:
                self.close()
                raise
            assert response.will_close != _UNKNOWN
            self.__state = _CS_IDLE

            if response.will_close:
                
                self.close()
            else:
                
                self.__response = response

            return response
        except:
            response.close()
            raise

try:
    import ssl
except ImportError:
    pass
else:
    class HTTPSConnection(HTTPConnection):
        "This class allows communication via SSL."

        default_port = HTTPS_PORT

        def __init__(self, host, port=None,
                     *, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                     source_address=None, context=None, blocksize=8192):
            super(HTTPSConnection, self).__init__(host, port, timeout,
                                                  source_address,
                                                  blocksize=blocksize)
            if context is None:
                context = _create_https_context(self._http_vsn)
            self._context = context

        def connect(self):
            "Connect to a host on a given (SSL) port."

            super().connect()

            if self._tunnel_host:
                server_hostname = self._tunnel_host
            else:
                server_hostname = self.host

            self.sock = self._context.wrap_socket(self.sock,
                                                  server_hostname=server_hostname)

    __all__.append("HTTPSConnection")

class HTTPException(Exception):
    
    
    pass

class NotConnected(HTTPException):
    pass

class InvalidURL(HTTPException):
    pass

class UnknownProtocol(HTTPException):
    def __init__(self, version):
        self.args = version,
        self.version = version

class UnknownTransferEncoding(HTTPException):
    pass

class UnimplementedFileMode(HTTPException):
    pass

class IncompleteRead(HTTPException):
    def __init__(self, partial, expected=None):
        self.args = partial,
        self.partial = partial
        self.expected = expected
    def __repr__(self):
        if self.expected is not None:
            e = ', %i more expected' % self.expected
        else:
            e = ''
        return '%s(%i bytes read%s)' % (self.__class__.__name__,
                                        len(self.partial), e)
    __str__ = object.__str__

class ImproperConnectionState(HTTPException):
    pass

class CannotSendRequest(ImproperConnectionState):
    pass

class CannotSendHeader(ImproperConnectionState):
    pass

class ResponseNotReady(ImproperConnectionState):
    pass

class BadStatusLine(HTTPException):
    def __init__(self, line):
        if not line:
            line = repr(line)
        self.args = line,
        self.line = line

class LineTooLong(HTTPException):
    def __init__(self, line_type):
        HTTPException.__init__(self, "got more than %d bytes when reading %s"
                                     % (_MAXLINE, line_type))

class RemoteDisconnected(ConnectionResetError, BadStatusLine):
    def __init__(self, *pos, **kw):
        BadStatusLine.__init__(self, "")
        ConnectionResetError.__init__(self, *pos, **kw)


error = HTTPException