| from typing import Dict, Generator | |
| from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response | |
| from pip._internal.exceptions import NetworkConnectionError | |
| # The following comments and HTTP headers were originally added by | |
| # Donald Stufft in git commit 22c562429a61bb77172039e480873fb239dd8c03. | |
| # | |
| # We use Accept-Encoding: identity here because requests defaults to | |
| # accepting compressed responses. This breaks in a variety of ways | |
| # depending on how the server is configured. | |
| # - Some servers will notice that the file isn't a compressible file | |
| # and will leave the file alone and with an empty Content-Encoding | |
| # - Some servers will notice that the file is already compressed and | |
| # will leave the file alone, adding a Content-Encoding: gzip header | |
| # - Some servers won't notice anything at all and will take a file | |
| # that's already been compressed and compress it again, and set | |
| # the Content-Encoding: gzip header | |
| # By setting this to request only the identity encoding we're hoping | |
| # to eliminate the third case. Hopefully there does not exist a server | |
| # which when given a file will notice it is already compressed and that | |
| # you're not asking for a compressed file and will then decompress it | |
| # before sending because if that's the case I don't think it'll ever be | |
| # possible to make this work. | |
| HEADERS: Dict[str, str] = {"Accept-Encoding": "identity"} | |
| def raise_for_status(resp: Response) -> None: | |
| http_error_msg = "" | |
| if isinstance(resp.reason, bytes): | |
| # We attempt to decode utf-8 first because some servers | |
| # choose to localize their reason strings. If the string | |
| # isn't utf-8, we fall back to iso-8859-1 for all other | |
| # encodings. | |
| try: | |
| reason = resp.reason.decode("utf-8") | |
| except UnicodeDecodeError: | |
| reason = resp.reason.decode("iso-8859-1") | |
| else: | |
| reason = resp.reason | |
| if 400 <= resp.status_code < 500: | |
| http_error_msg = ( | |
| f"{resp.status_code} Client Error: {reason} for url: {resp.url}" | |
| ) | |
| elif 500 <= resp.status_code < 600: | |
| http_error_msg = ( | |
| f"{resp.status_code} Server Error: {reason} for url: {resp.url}" | |
| ) | |
| if http_error_msg: | |
| raise NetworkConnectionError(http_error_msg, response=resp) | |
| def response_chunks( | |
| response: Response, chunk_size: int = CONTENT_CHUNK_SIZE | |
| ) -> Generator[bytes, None, None]: | |
| """Given a requests Response, provide the data chunks.""" | |
| try: | |
| # Special case for urllib3. | |
| for chunk in response.raw.stream( | |
| chunk_size, | |
| # We use decode_content=False here because we don't | |
| # want urllib3 to mess with the raw bytes we get | |
| # from the server. If we decompress inside of | |
| # urllib3 then we cannot verify the checksum | |
| # because the checksum will be of the compressed | |
| # file. This breakage will only occur if the | |
| # server adds a Content-Encoding header, which | |
| # depends on how the server was configured: | |
| # - Some servers will notice that the file isn't a | |
| # compressible file and will leave the file alone | |
| # and with an empty Content-Encoding | |
| # - Some servers will notice that the file is | |
| # already compressed and will leave the file | |
| # alone and will add a Content-Encoding: gzip | |
| # header | |
| # - Some servers won't notice anything at all and | |
| # will take a file that's already been compressed | |
| # and compress it again and set the | |
| # Content-Encoding: gzip header | |
| # | |
| # By setting this not to decode automatically we | |
| # hope to eliminate problems with the second case. | |
| decode_content=False, | |
| ): | |
| yield chunk | |
| except AttributeError: | |
| # Standard file-like object. | |
| while True: | |
| chunk = response.raw.read(chunk_size) | |
| if not chunk: | |
| break | |
| yield chunk | |