xxx this test doesn t seem to test anything useful clear opener global variable tests urllib request urlopen using the network these tests are not exhaustive assuming that testing using files does a good job overall of some of the basic interface features there are no tests exercising the optional data and proxies arguments no tests for transparent redirection have been written setup is not used for always constructing a connection to http www pythontest net since there a few tests that don t use that address and making a connection is expensive enough to warrant minimizing unneeded connections clear opener global variable simple test expected to pass test both readline and readlines test info make sure same url as opened is returned by geturl test getcode with the fancy opener to get 404 error codes make sure proper exception is raised when connecting to a bogus address given that both verisign and various isps have in the past or are presently hijacking various invalid domain name requests in an attempt to boost traffic to their own sites finding a domain name to use for this test is difficult rfc2606 leads one to believe that invalid should work but experience seemed to indicate otherwise single character tlds are likely to remain invalid so this seems to be the best choice the trailing prevents a related problem the normal dns resolver appends the domain names from the search path if there is no the end and and if one of those domains implements a rule a result is returned however none of this will prevent the test from failing if the isp hijacks all invalid domain requests the real solution would be to be able to parameterize the framework with a mock resolver socket gaierror is too narrow since getaddrinfo may also fail with eaisystem and etimedout seen on ubuntu 13 04 i e python s timeouterror this happens with some overzealous dns providers such as opendns tests urllib request urlretrieve using the network def setupself remove temporary files urlretrieve self addcleanupurllib request urlcleanup contextlib contextmanager def urlretrieveself args kwargs resource args0 with sockethelper transientinternetresource filelocation info urllib request urlretrieveargs kwargs try yield filelocation info finally oshelper unlinkfilelocation def testbasicself test basic functionality with self urlretrieveself logo as filelocation info self asserttrueos path existsfilelocation file location returned by urlretrieve is not a valid path with openfilelocation rb as f self asserttruef read reading from the file location returned by urlretrieve failed def testspecifiedpathself make sure that specifying the location of the file to write to works with self urlretrieveself logo oshelper testfn as filelocation info self assertequalfilelocation oshelper testfn self asserttrueos path existsfilelocation with openfilelocation rb as f self asserttruef read reading from temporary file failed def testheaderself make sure header returned as 2nd value from urlretrieve is good with self urlretrieveself logo as filelocation info self assertisinstanceinfo email message message info is not an instance of email message message logo http www pythontest net support requiresresource walltime def testdataheaderself with self urlretrieveself logo as filelocation fileheaders datevalue fileheaders get date dateformat a d b y h m s gmt try time strptimedatevalue dateformat except valueerror self fail date value not in r format dateformat def testreporthookself records def recordingreporthookblocks blocksize totalsize records appendblocks blocksize totalsize with self urlretrieveself logo reporthookrecordingreporthook as filelocation fileheaders expectedsize intfileheaders contentlength recordsrepr reprrecords for use in error messages self assertgreaterlenrecords 1 msgthere should always be two calls the first one before the transfer starts self assertequalrecords00 0 self assertgreaterrecords01 0 msgblock size can t be 0 in s recordsrepr self assertequalrecords02 expectedsize self assertequalrecords12 expectedsize blocksizes blocksize for blocksize in records self assertequalrecords01 blocksizes msgblock sizes in s must be equal recordsrepr self assertgreaterequalrecords10records01 expectedsize msgnumber of blocks block size must be total size in s recordsrepr if name main unittest main xxx this test doesn t seem to test anything useful clear _opener global variable tests urllib request urlopen using the network these tests are not exhaustive assuming that testing using files does a good job overall of some of the basic interface features there are no tests exercising the optional data and proxies arguments no tests for transparent redirection have been written setup is not used for always constructing a connection to http www pythontest net since there a few tests that don t use that address and making a connection is expensive enough to warrant minimizing unneeded connections clear _opener global variable simple test expected to pass test both readline and readlines test info make sure same url as opened is returned by geturl test getcode with the fancy opener to get 404 error codes make sure proper exception is raised when connecting to a bogus address given that both verisign and various isps have in the past or are presently hijacking various invalid domain name requests in an attempt to boost traffic to their own sites finding a domain name to use for this test is difficult rfc2606 leads one to believe that invalid should work but experience seemed to indicate otherwise single character tlds are likely to remain invalid so this seems to be the best choice the trailing prevents a related problem the normal dns resolver appends the domain names from the search path if there is no the end and and if one of those domains implements a rule a result is returned however none of this will prevent the test from failing if the isp hijacks all invalid domain requests the real solution would be to be able to parameterize the framework with a mock resolver socket gaierror is too narrow since getaddrinfo may also fail with eai_system and etimedout seen on ubuntu 13 04 i e python s timeouterror this happens with some overzealous dns providers such as opendns tests urllib request urlretrieve using the network remove temporary files urlretrieve test basic functionality make sure that specifying the location of the file to write to works make sure header returned as 2nd value from urlretrieve is good for use in error messages
import unittest
from test import support
from test.support import os_helper
from test.support import socket_helper

import contextlib
import socket
import urllib.parse
import urllib.request
import os
import email.message
import time


support.requires('network')


class URLTimeoutTest(unittest.TestCase):
    

    def setUp(self):
        socket.setdefaulttimeout(support.INTERNET_TIMEOUT)

    def tearDown(self):
        socket.setdefaulttimeout(None)

    def testURLread(self):
        
        self.addCleanup(urllib.request.urlcleanup)

        domain = urllib.parse.urlparse(support.TEST_HTTP_URL).netloc
        with socket_helper.transient_internet(domain):
            f = urllib.request.urlopen(support.TEST_HTTP_URL)
            f.read()


class urlopenNetworkTests(unittest.TestCase):
    

    url = 'http://www.pythontest.net/'

    def setUp(self):
        
        self.addCleanup(urllib.request.urlcleanup)

    @contextlib.contextmanager
    def urlopen(self, *args, **kwargs):
        resource = args[0]
        with socket_helper.transient_internet(resource):
            r = urllib.request.urlopen(*args, **kwargs)
            try:
                yield r
            finally:
                r.close()

    def test_basic(self):
        
        with self.urlopen(self.url) as open_url:
            for attr in ("read", "readline", "readlines", "fileno", "close",
                         "info", "geturl"):
                self.assertTrue(hasattr(open_url, attr), "object returned from "
                                "urlopen lacks the %s attribute" % attr)
            self.assertTrue(open_url.read(), "calling 'read' failed")

    def test_readlines(self):
        
        with self.urlopen(self.url) as open_url:
            self.assertIsInstance(open_url.readline(), bytes,
                                  "readline did not return a string")
            self.assertIsInstance(open_url.readlines(), list,
                                  "readlines did not return a list")

    def test_info(self):
        
        with self.urlopen(self.url) as open_url:
            info_obj = open_url.info()
            self.assertIsInstance(info_obj, email.message.Message,
                                  "object returned by 'info' is not an "
                                  "instance of email.message.Message")
            self.assertEqual(info_obj.get_content_subtype(), "html")

    def test_geturl(self):
        
        with self.urlopen(self.url) as open_url:
            gotten_url = open_url.geturl()
            self.assertEqual(gotten_url, self.url)

    def test_getcode(self):
        
        URL = self.url + "XXXinvalidXXX"
        with socket_helper.transient_internet(URL):
            with self.assertWarns(DeprecationWarning):
                open_url = urllib.request.FancyURLopener().open(URL)
            try:
                code = open_url.getcode()
            finally:
                open_url.close()
            self.assertEqual(code, 404)

    @support.requires_resource('walltime')
    def test_bad_address(self):
        
        

        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        bogus_domain = "sadflkjsasf.i.nvali.d."
        try:
            socket.gethostbyname(bogus_domain)
        except OSError:
            
            
            
            pass
        else:
            
            self.skipTest("%r should not resolve for test to work" % bogus_domain)
        failure_explanation = ('opening an invalid URL did not raise OSError; '
                               'can be caused by a broken DNS server '
                               '(e.g. returns 404 or hijacks page)')
        with self.assertRaises(OSError, msg=failure_explanation):
            urllib.request.urlopen("http://{}/".format(bogus_domain))


class urlretrieveNetworkTests(unittest.TestCase):
    

    def setUp(self):
        
        self.addCleanup(urllib.request.urlcleanup)

    @contextlib.contextmanager
    def urlretrieve(self, *args, **kwargs):
        resource = args[0]
        with socket_helper.transient_internet(resource):
            file_location, info = urllib.request.urlretrieve(*args, **kwargs)
            try:
                yield file_location, info
            finally:
                os_helper.unlink(file_location)

    def test_basic(self):
        
        with self.urlretrieve(self.logo) as (file_location, info):
            self.assertTrue(os.path.exists(file_location), "file location returned by"
                            " urlretrieve is not a valid path")
            with open(file_location, 'rb') as f:
                self.assertTrue(f.read(), "reading from the file location returned"
                                " by urlretrieve failed")

    def test_specified_path(self):
        
        with self.urlretrieve(self.logo,
                              os_helper.TESTFN) as (file_location, info):
            self.assertEqual(file_location, os_helper.TESTFN)
            self.assertTrue(os.path.exists(file_location))
            with open(file_location, 'rb') as f:
                self.assertTrue(f.read(), "reading from temporary file failed")

    def test_header(self):
        
        with self.urlretrieve(self.logo) as (file_location, info):
            self.assertIsInstance(info, email.message.Message,
                                  "info is not an instance of email.message.Message")

    logo = "http://www.pythontest.net/"

    @support.requires_resource('walltime')
    def test_data_header(self):
        with self.urlretrieve(self.logo) as (file_location, fileheaders):
            datevalue = fileheaders.get('Date')
            dateformat = '%a, %d %b %Y %H:%M:%S GMT'
            try:
                time.strptime(datevalue, dateformat)
            except ValueError:
                self.fail('Date value not in %r format' % dateformat)

    def test_reporthook(self):
        records = []

        def recording_reporthook(blocks, block_size, total_size):
            records.append((blocks, block_size, total_size))

        with self.urlretrieve(self.logo, reporthook=recording_reporthook) as (
                file_location, fileheaders):
            expected_size = int(fileheaders['Content-Length'])

        records_repr = repr(records)  
        self.assertGreater(len(records), 1, msg="There should always be two "
                           "calls; the first one before the transfer starts.")
        self.assertEqual(records[0][0], 0)
        self.assertGreater(records[0][1], 0,
                           msg="block size can't be 0 in %s" % records_repr)
        self.assertEqual(records[0][2], expected_size)
        self.assertEqual(records[-1][2], expected_size)

        block_sizes = {block_size for _, block_size, _ in records}
        self.assertEqual({records[0][1]}, block_sizes,
                         msg="block sizes in %s must be equal" % records_repr)
        self.assertGreaterEqual(records[-1][0]*records[0][1], expected_size,
                                msg="number of blocks * block size must be"
                                " >= total size in %s" % records_repr)


if __name__ == "__main__":
    unittest.main()