Python test.support.transient_internet() Examples

def wrap_methods(cls):
        # Wrap all methods in a transient_internet() exception catcher
        # XXX put a generic version in test.support?
        def wrap_meth(meth):
            @functools.wraps(meth)
            def wrapped(self):
                with support.transient_internet(self.NNTP_HOST):
                    meth(self)
            return wrapped
        for name in dir(cls):
            if not name.startswith('test_'):
                continue
            meth = getattr(cls, name)
            if not callable(meth):
                continue
            # Need to use a closure so that meth remains bound to its current
            # value
            setattr(cls, name, wrap_meth(meth)) 

def test_connect_cadata(self):
        with open(CAFILE_CACERT) as f:
            pem = f.read()
        der = ssl.PEM_cert_to_DER_cert(pem)
        with support.transient_internet("svn.python.org"):
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            ctx.verify_mode = ssl.CERT_REQUIRED
            ctx.load_verify_locations(cadata=pem)
            with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
                s.connect(("svn.python.org", 443))
                cert = s.getpeercert()
                self.assertTrue(cert)

            # same with DER
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            ctx.verify_mode = ssl.CERT_REQUIRED
            ctx.load_verify_locations(cadata=der)
            with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
                s.connect(("svn.python.org", 443))
                cert = s.getpeercert()
                self.assertTrue(cert) 

def test_makefile_close(self):
        # Issue #5238: creating a file-like object with makefile() shouldn't
        # delay closing the underlying "real socket" (here tested with its
        # file descriptor, hence skipping the test under Windows).
        with support.transient_internet("svn.python.org"):
            ss = ssl.wrap_socket(socket.socket(socket.AF_INET))
            ss.connect(("svn.python.org", 443))
            fd = ss.fileno()
            f = ss.makefile()
            f.close()
            # The fd is still open
            os.read(fd, 0)
            # Closing the SSL socket should close the fd too
            ss.close()
            gc.collect()
            with self.assertRaises(OSError) as e:
                os.read(fd, 0)
            self.assertEqual(e.exception.errno, errno.EBADF) 

def wrap_methods(cls):
        # Wrap all methods in a transient_internet() exception catcher
        # XXX put a generic version in test.support?
        def wrap_meth(meth):
            @functools.wraps(meth)
            def wrapped(self):
                with support.transient_internet(self.NNTP_HOST):
                    meth(self)
            return wrapped
        for name in dir(cls):
            if not name.startswith('test_'):
                continue
            meth = getattr(cls, name)
            if not callable(meth):
                continue
            # Need to use a closure so that meth remains bound to its current
            # value
            setattr(cls, name, wrap_meth(meth)) 

def test_makefile_close(self):
        # Issue #5238: creating a file-like object with makefile() shouldn't
        # delay closing the underlying "real socket" (here tested with its
        # file descriptor, hence skipping the test under Windows).
        with support.transient_internet(REMOTE_HOST):
            ss = ssl.wrap_socket(socket.socket(socket.AF_INET))
            ss.connect((REMOTE_HOST, 443))
            fd = ss.fileno()
            f = ss.makefile()
            f.close()
            # The fd is still open
            os.read(fd, 0)
            # Closing the SSL socket should close the fd too
            ss.close()
            gc.collect()
            with self.assertRaises(OSError) as e:
                os.read(fd, 0)
            self.assertEqual(e.exception.errno, errno.EBADF) 

def test_connect_cadata(self):
        with open(REMOTE_ROOT_CERT) as f:
            pem = f.read()
        der = ssl.PEM_cert_to_DER_cert(pem)
        with support.transient_internet(REMOTE_HOST):
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            ctx.verify_mode = ssl.CERT_REQUIRED
            ctx.load_verify_locations(cadata=pem)
            with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
                s.connect((REMOTE_HOST, 443))
                cert = s.getpeercert()
                self.assertTrue(cert)

            # same with DER
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            ctx.verify_mode = ssl.CERT_REQUIRED
            ctx.load_verify_locations(cadata=der)
            with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
                s.connect((REMOTE_HOST, 443))
                cert = s.getpeercert()
                self.assertTrue(cert) 

def test_read_write_data(self):
        with support.transient_internet("svn.python.org"):
            sock = socket.socket(socket.AF_INET)
            sock.connect(("svn.python.org", 443))
            incoming = ssl.MemoryBIO()
            outgoing = ssl.MemoryBIO()
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            ctx.verify_mode = ssl.CERT_NONE
            sslobj = ctx.wrap_bio(incoming, outgoing, False)
            self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake)
            req = b'GET / HTTP/1.0\r\n\r\n'
            self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req)
            buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024)
            self.assertEqual(buf[:5], b'HTTP/')
            self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
            sock.close() 

def test_connect_ex_error(self):
        with support.transient_internet(REMOTE_HOST):
            s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                cert_reqs=ssl.CERT_REQUIRED,
                                ca_certs=REMOTE_ROOT_CERT)
            try:
                rc = s.connect_ex((REMOTE_HOST, 444))
                # Issue #19919: Windows machines or VMs hosted on Windows
                # machines sometimes return EWOULDBLOCK.
                errors = (
                    errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT,
                    errno.EWOULDBLOCK,
                )
                self.assertIn(rc, errors)
            finally:
                s.close() 

def test_ciphers(self):
        remote = (REMOTE_HOST, 443)
        with support.transient_internet(remote[0]):
            with ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s:
                s.connect(remote)
            with ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s:
                s.connect(remote)
            # Error checking can happen at instantiation or when connecting
            with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"):
                with socket.socket(socket.AF_INET) as sock:
                    s = ssl.wrap_socket(sock,
                                        cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx")
                    s.connect(remote) 

def test_http_timeout(self):
        url = "http://www.example.com"
        with support.transient_internet(url):
            u = _urlopen_with_retry(url, timeout=120)
            self.addCleanup(u.close)
            self.assertEqual(u.fp.raw._sock.gettimeout(), 120) 

def test_ftp_default_timeout(self):
        self.assertIsNone(socket.getdefaulttimeout())
        with support.transient_internet(self.FTP_HOST):
            socket.setdefaulttimeout(60)
            try:
                u = _urlopen_with_retry(self.FTP_HOST)
                self.addCleanup(u.close)
            finally:
                socket.setdefaulttimeout(None)
            self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) 

def test_ftp_no_timeout(self):
        self.assertIsNone(socket.getdefaulttimeout())
        with support.transient_internet(self.FTP_HOST):
            socket.setdefaulttimeout(60)
            try:
                u = _urlopen_with_retry(self.FTP_HOST, timeout=None)
                self.addCleanup(u.close)
            finally:
                socket.setdefaulttimeout(None)
            self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) 

def test_ftp_timeout(self):
        with support.transient_internet(self.FTP_HOST):
            u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
            self.addCleanup(u.close)
            self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) 

def test_get_ca_certs_capath(self):
        # capath certs are loaded on request
        with support.transient_internet(REMOTE_HOST):
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            ctx.verify_mode = ssl.CERT_REQUIRED
            ctx.load_verify_locations(capath=CAPATH)
            self.assertEqual(ctx.get_ca_certs(), [])
            s = ctx.wrap_socket(socket.socket(socket.AF_INET))
            s.connect((REMOTE_HOST, 443))
            try:
                cert = s.getpeercert()
                self.assertTrue(cert)
            finally:
                s.close()
            self.assertEqual(len(ctx.get_ca_certs()), 1) 

def test_non_blocking_connect_ex(self):
        # Issue #11326: non-blocking connect_ex() should allow handshake
        # to proceed after the socket gets ready.
        with support.transient_internet(REMOTE_HOST):
            s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                cert_reqs=ssl.CERT_REQUIRED,
                                ca_certs=REMOTE_ROOT_CERT,
                                do_handshake_on_connect=False)
            try:
                s.setblocking(False)
                rc = s.connect_ex((REMOTE_HOST, 443))
                # EWOULDBLOCK under Windows, EINPROGRESS elsewhere
                self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK))
                # Wait for connect to finish
                select.select([], [s], [], 5.0)
                # Non-blocking handshake
                while True:
                    try:
                        s.do_handshake()
                        break
                    except ssl.SSLWantReadError:
                        select.select([s], [], [], 5.0)
                    except ssl.SSLWantWriteError:
                        select.select([], [s], [], 5.0)
                # SSL established
                self.assertTrue(s.getpeercert())
            finally:
                s.close() 

def test_connect_with_context(self):
        with support.transient_internet(REMOTE_HOST):
            # Same as test_connect, but with a separately created context
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            s = ctx.wrap_socket(socket.socket(socket.AF_INET))
            s.connect((REMOTE_HOST, 443))
            try:
                self.assertEqual({}, s.getpeercert())
            finally:
                s.close()
            # Same with a server hostname
            s = ctx.wrap_socket(socket.socket(socket.AF_INET),
                                server_hostname=REMOTE_HOST)
            s.connect((REMOTE_HOST, 443))
            s.close()
            # This should fail because we have no verification certs
            ctx.verify_mode = ssl.CERT_REQUIRED
            s = ctx.wrap_socket(socket.socket(socket.AF_INET))
            self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
                                    s.connect, (REMOTE_HOST, 443))
            s.close()
            # This should succeed because we specify the root cert
            ctx.load_verify_locations(REMOTE_ROOT_CERT)
            s = ctx.wrap_socket(socket.socket(socket.AF_INET))
            s.connect((REMOTE_HOST, 443))
            try:
                cert = s.getpeercert()
                self.assertTrue(cert)
            finally:
                s.close() 

def test_connect_capath(self):
        # Verify server certificates using the `capath` argument
        # NOTE: the subject hashing algorithm has been changed between
        # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must
        # contain both versions of each certificate (same content, different
        # filename) for this test to be portable across OpenSSL releases.
        with support.transient_internet(REMOTE_HOST):
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            ctx.verify_mode = ssl.CERT_REQUIRED
            ctx.load_verify_locations(capath=CAPATH)
            s = ctx.wrap_socket(socket.socket(socket.AF_INET))
            s.connect((REMOTE_HOST, 443))
            try:
                cert = s.getpeercert()
                self.assertTrue(cert)
            finally:
                s.close()
            # Same with a bytes `capath` argument
            ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            ctx.verify_mode = ssl.CERT_REQUIRED
            ctx.load_verify_locations(capath=BYTES_CAPATH)
            s = ctx.wrap_socket(socket.socket(socket.AF_INET))
            s.connect((REMOTE_HOST, 443))
            try:
                cert = s.getpeercert()
                self.assertTrue(cert)
            finally:
                s.close() 

def test_get_server_certificate(self):
        def _test_get_server_certificate(host, port, cert=None):
            with support.transient_internet(host):
                pem = ssl.get_server_certificate((host, port),
                                                 ssl.PROTOCOL_SSLv23)
                if not pem:
                    self.fail("No server certificate on %s:%s!" % (host, port))

                try:
                    pem = ssl.get_server_certificate((host, port),
                                                     ssl.PROTOCOL_SSLv23,
                                                     ca_certs=CERTFILE)
                except ssl.SSLError as x:
                    #should fail
                    if support.verbose:
                        sys.stdout.write("%s\n" % x)
                else:
                    self.fail("Got server certificate %s for %s:%s!" % (pem, host, port))

                pem = ssl.get_server_certificate((host, port),
                                                 ssl.PROTOCOL_SSLv23,
                                                 ca_certs=cert)
                if not pem:
                    self.fail("No server certificate on %s:%s!" % (host, port))
                if support.verbose:
                    sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem))

        _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT)
        if support.IPV6_ENABLED:
            _test_get_server_certificate('ipv6.google.com', 443) 

def urlretrieve(self, *args, **kwargs):
        resource = args[0]
        with support.transient_internet(resource):
            file_location, info = urllib.request.urlretrieve(*args, **kwargs)
            try:
                yield file_location, info
            finally:
                support.unlink(file_location) 

def test_http_no_timeout(self):
        self.assertIsNone(socket.getdefaulttimeout())
        url = "http://www.example.com"
        with support.transient_internet(url):
            socket.setdefaulttimeout(60)
            try:
                u = _urlopen_with_retry(url, timeout=None)
                self.addCleanup(u.close)
            finally:
                socket.setdefaulttimeout(None)
            self.assertIsNone(u.fp.raw._sock.gettimeout()) 

def test_http_default_timeout(self):
        self.assertIsNone(socket.getdefaulttimeout())
        url = "http://www.example.com"
        with support.transient_internet(url):
            socket.setdefaulttimeout(60)
            try:
                u = _urlopen_with_retry(url)
                self.addCleanup(u.close)
            finally:
                socket.setdefaulttimeout(None)
            self.assertEqual(u.fp.raw._sock.gettimeout(), 60) 

def test_http_basic(self):
        self.assertIsNone(socket.getdefaulttimeout())
        url = "http://www.example.com"
        with support.transient_internet(url, timeout=None):
            u = _urlopen_with_retry(url)
            self.addCleanup(u.close)
            self.assertIsNone(u.fp.raw._sock.gettimeout()) 

def _test_urls(self, urls, handlers, retry=True):
        import time
        import logging
        debug = logging.getLogger("test_urllib2").debug

        urlopen = urllib.request.build_opener(*handlers).open
        if retry:
            urlopen = _wrap_with_retry_thrice(urlopen, urllib.error.URLError)

        for url in urls:
            with self.subTest(url=url):
                if isinstance(url, tuple):
                    url, req, expected_err = url
                else:
                    req = expected_err = None

                with support.transient_internet(url):
                    try:
                        f = urlopen(url, req, TIMEOUT)
                    # urllib.error.URLError is a subclass of OSError
                    except OSError as err:
                        if expected_err:
                            msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" %
                                   (expected_err, url, req, type(err), err))
                            self.assertIsInstance(err, expected_err, msg)
                        else:
                            raise
                    else:
                        try:
                            with support.time_out, \
                                 support.socket_peer_reset, \
                                 support.ioerror_peer_reset:
                                buf = f.read()
                                debug("read %d bytes" % len(buf))
                        except socket.timeout:
                            print("<timeout: %s>" % url, file=sys.stderr)
                        f.close()
                time.sleep(0.1) 

def test_custom_headers(self):
        url = "http://www.example.com"
        with support.transient_internet(url):
            opener = urllib.request.build_opener()
            request = urllib.request.Request(url)
            self.assertFalse(request.header_items())
            opener.open(request)
            self.assertTrue(request.header_items())
            self.assertTrue(request.has_header('User-agent'))
            request.add_header('User-Agent','Test-Agent')
            opener.open(request)
            self.assertEqual(request.get_header('User-agent'),'Test-Agent') 

def test_redirect_url_withfrag(self):
        redirect_url_with_frag = "http://www.pythontest.net/redir/with_frag/"
        with support.transient_internet(redirect_url_with_frag):
            req = urllib.request.Request(redirect_url_with_frag)
            res = urllib.request.urlopen(req)
            self.assertEqual(res.geturl(),
                    "http://www.pythontest.net/elsewhere/#frag") 

def test_urlwithfrag(self):
        urlwith_frag = "http://www.pythontest.net/index.html#frag"
        with support.transient_internet(urlwith_frag):
            req = urllib.request.Request(urlwith_frag)
            res = urllib.request.urlopen(req)
            self.assertEqual(res.geturl(),
                    "http://www.pythontest.net/index.html#frag") 

def test_close(self):
        # calling .close() on urllib2's response objects should close the
        # underlying socket
        url = "http://www.example.com/"
        with support.transient_internet(url):
            response = _urlopen_with_retry(url)
            sock = response.fp
            self.assertFalse(sock.closed)
            response.close()
            self.assertTrue(sock.closed) 

def test_idna(self):
        # Check for internet access before running test
        # (issue #12804, issue #25138).
        with support.transient_internet('python.org'):
            socket.gethostbyname('python.org')

        # these should all be successful
        domain = 'испытание.pythontest.net'
        socket.gethostbyname(domain)
        socket.gethostbyname_ex(domain)
        socket.getaddrinfo(domain,0,socket.AF_UNSPEC,socket.SOCK_STREAM)
        # this may not work if the forward lookup choses the IPv6 address, as that doesn't
        # have a reverse entry yet
        # socket.gethostbyaddr('испытание.python.org') 

def testRecvTimeout(self):
        # Test recv() timeout
        with support.transient_internet(self.addr_remote[0]):
            self.sock.connect(self.addr_remote)
            self._sock_operation(1, 1.5, 'recv', 1024) 

def resolve_address(host, port):
    """Resolve an (host, port) to an address.

    We must perform name resolution before timeout tests, otherwise it will be
    performed by connect().
    """
    with support.transient_internet(host):
        return socket.getaddrinfo(host, port, socket.AF_INET,
                                  socket.SOCK_STREAM)[0][4]