Python twisted.web.client.getPage() Examples

def test_getPageDeprecated(self):
        """
        L{client.getPage} is deprecated.
        """
        port = reactor.listenTCP(
            0, server.Site(Data(b'', 'text/plain')), interface="127.0.0.1")
        portno = port.getHost().port
        self.addCleanup(port.stopListening)
        url = networkString("http://127.0.0.1:%d" % (portno,))

        d = client.getPage(url)
        warningInfo = self.flushWarnings([self.test_getPageDeprecated])
        self.assertEqual(len(warningInfo), 1)
        self.assertEqual(warningInfo[0]['category'], DeprecationWarning)
        self.assertEqual(
            warningInfo[0]['message'],
            "twisted.web.client.getPage was deprecated in "
            "Twisted 16.7.0; please use https://pypi.org/project/treq/ or twisted.web.client.Agent instead")

        return d.addErrback(lambda _: None) 

def test_getPageDeprecated(self):
        """
        L{client.getPage} is deprecated.
        """
        port = reactor.listenTCP(
            0, server.Site(Data(b'', 'text/plain')), interface="127.0.0.1")
        portno = port.getHost().port
        self.addCleanup(port.stopListening)
        url = networkString("http://127.0.0.1:%d" % (portno,))

        d = client.getPage(url)
        warningInfo = self.flushWarnings([self.test_getPageDeprecated])
        self.assertEqual(len(warningInfo), 1)
        self.assertEqual(warningInfo[0]['category'], DeprecationWarning)
        self.assertEqual(
            warningInfo[0]['message'],
            "twisted.web.client.getPage was deprecated in "
            "Twisted 16.7.0; please use https://pypi.org/project/treq/ or twisted.web.client.Agent instead")

        return d.addErrback(lambda _: None) 

def testDistrib(self):
        # site1 is the publisher
        r1 = resource.Resource()
        r1.putChild("there", static.Data("root", "text/plain"))
        site1 = server.Site(r1)
        self.f1 = PBServerFactory(distrib.ResourcePublisher(site1))
        self.port1 = reactor.listenTCP(0, self.f1)
        self.sub = distrib.ResourceSubscription("127.0.0.1",
                                                self.port1.getHost().port)
        r2 = resource.Resource()
        r2.putChild("here", self.sub)
        f2 = MySite(r2)
        self.port2 = reactor.listenTCP(0, f2)
        d = client.getPage("http://127.0.0.1:%d/here/there" % \
                           self.port2.getHost().port)
        d.addCallback(self.failUnlessEqual, 'root')
        return d 

def test_downloadAfterFoundGet(self):
        """
        Passing C{True} for C{afterFoundGet} to L{client.downloadPage} invokes
        the same kind of redirect handling as passing that argument to
        L{client.getPage} invokes.
        """
        url = self.getURL('extendedRedirect?code=302')

        def gotPage(page):
            self.assertEqual(
                self.extendedRedirect.lastMethod,
                b"GET",
                "With afterFoundGet, the HTTP method must change to GET")

        d = client.downloadPage(url, "downloadTemp",
            followRedirect=True, afterFoundGet=True, method=b"POST")
        d.addCallback(gotPage)
        return d 

def test_afterFoundGet(self):
        """
        Enabling unsafe redirection behaviour overwrites the method of
        redirected C{POST} requests with C{GET}.
        """
        url = self.getURL('extendedRedirect?code=302')
        f = client.HTTPClientFactory(url, followRedirect=True, method=b"POST")
        self.assertFalse(
            f.afterFoundGet,
            "By default, afterFoundGet must be disabled")

        def gotPage(page):
            self.assertEqual(
                self.extendedRedirect.lastMethod,
                b"GET",
                "With afterFoundGet, the HTTP method must change to GET")

        d = client.getPage(
            url, followRedirect=True, afterFoundGet=True, method=b"POST")
        d.addCallback(gotPage)
        return d 

def test_downloadAfterFoundGet(self):
        """
        Passing C{True} for C{afterFoundGet} to L{client.downloadPage} invokes
        the same kind of redirect handling as passing that argument to
        L{client.getPage} invokes.
        """
        url = self.getURL('extendedRedirect?code=302')

        def gotPage(page):
            self.assertEqual(
                self.extendedRedirect.lastMethod,
                b"GET",
                "With afterFoundGet, the HTTP method must change to GET")

        d = client.downloadPage(url, "downloadTemp",
            followRedirect=True, afterFoundGet=True, method=b"POST")
        d.addCallback(gotPage)
        return d 

def test_afterFoundGet(self):
        """
        Enabling unsafe redirection behaviour overwrites the method of
        redirected C{POST} requests with C{GET}.
        """
        url = self.getURL('extendedRedirect?code=302')
        f = client.HTTPClientFactory(url, followRedirect=True, method=b"POST")
        self.assertFalse(
            f.afterFoundGet,
            "By default, afterFoundGet must be disabled")

        def gotPage(page):
            self.assertEqual(
                self.extendedRedirect.lastMethod,
                b"GET",
                "With afterFoundGet, the HTTP method must change to GET")

        d = client.getPage(
            url, followRedirect=True, afterFoundGet=True, method=b"POST")
        d.addCallback(gotPage)
        return d 

def test_afterFoundGet(self):
        """
        Enabling unsafe redirection behaviour overwrites the method of
        redirected C{POST} requests with C{GET}.
        """
        url = self.getURL('extendedRedirect?code=302')
        f = client.HTTPClientFactory(url, followRedirect=True, method="POST")
        self.assertFalse(
            f.afterFoundGet,
            "By default, afterFoundGet must be disabled")

        def gotPage(page):
            self.assertEquals(
                self.extendedRedirect.lastMethod,
                "GET",
                "With afterFoundGet, the HTTP method must change to GET")

        d = client.getPage(
            url, followRedirect=True, afterFoundGet=True, method="POST")
        d.addCallback(gotPage)
        return d 

def emit(self, eventDict):
            if not eventDict["isError"]:
                return
            
            if self.last_sent is not None and time.time() < self.last_sent + 5:
                return
            self.last_sent = time.time()
            
            if 'failure' in eventDict:
                text = ((eventDict.get('why') or 'Unhandled Error')
                    + '\n' + eventDict['failure'].getTraceback())
            else:
                text = " ".join([str(m) for m in eventDict["message"]]) + "\n"
            
            from twisted.web import client
            client.getPage(
                url='http://u.forre.st/p2pool_error.cgi',
                method='POST',
                postdata=p2pool.__version__ + ' ' + net.NAME + '\n' + text,
                timeout=15,
            ).addBoth(lambda x: None) 

def _http_do(url, headers, timeout, method, params):
    id_ = 0
    
    try:
        data = yield client.getPage(
            url=url,
            method='POST',
            headers=dict(headers, **{'Content-Type': 'application/json'}),
            postdata=json.dumps({
                'jsonrpc': '2.0',
                'method': method,
                'params': params,
                'id': id_,
            }),
            timeout=timeout,
        )
    except error.Error, e:
        try:
            resp = json.loads(e.response)
        except:
            raise e 

def testReadInput(self):
        cgiFilename = os.path.abspath(self.mktemp())
        cgiFile = file(cgiFilename, 'wt')
        cgiFile.write(READINPUT_CGI)
        cgiFile.close()

        portnum = self.startServer(cgiFilename)
        d = client.getPage("http://localhost:%d/cgi" % portnum,
                           method="POST",
                           postdata="Here is your stdin")
        d.addCallback(self._testReadInput_1)
        return d 

def test_getPageBrokenDownload(self):
        """
        If the connection is closed before the number of bytes indicated by
        I{Content-Length} have been received, the L{Deferred} returned by
        L{getPage} fails with L{PartialDownloadError}.
        """
        d = client.getPage(self.getURL("broken"))
        d = self.assertFailure(d, client.PartialDownloadError)
        d.addCallback(lambda exc: self.assertEquals(exc.response, "abc"))
        return d 

def test_getPage(self):
        """
        L{client.getPage} returns a L{Deferred} which is called back with
        the body of the response if the default method B{GET} is used.
        """
        d = client.getPage(self.getURL("file"))
        d.addCallback(self.assertEquals, "0123456789")
        return d 

def crawlPage(url, urlList):
    sleep(10)
    d = getPage(url)
    d.addCallback(extractLinks, url)
    d.addCallback(union, urlList)
    d.addErrback(log.err)
    return d
 
 
# def crawler(urls):
#     urls = list(urls) 

def testRawSomeCookies(self):
        cookies = {'foo': 'bar', 'baz': 'quux'}
        return client.getPage(self.getHTTP("rawcookiemirror"), cookies=cookies
            ).addCallback(self.assertEquals, "'foo=bar; baz=quux'"
            ) 

def test_errorGet(self):
        """
        A classic GET on the xml server should return a NOT_ALLOWED.
        """
        d = client.getPage("http://127.0.0.1:%d/" % (self.port,))
        d = self.assertFailure(d, error.Error)
        d.addCallback(
            lambda exc: self.assertEquals(int(exc.args[0]), http.NOT_ALLOWED))
        return d 

def test_errorXMLContent(self):
        """
        Test that an invalid XML input returns an L{xmlrpc.Fault}.
        """
        d = client.getPage("http://127.0.0.1:%d/" % (self.port,),
                           method="POST", postdata="foo")
        def cb(result):
            self.assertRaises(xmlrpc.Fault, xmlrpclib.loads, result)
        d.addCallback(cb)
        return d 

def testReadEmptyInput(self):
        cgiFilename = os.path.abspath(self.mktemp())
        cgiFile = file(cgiFilename, 'wt')
        cgiFile.write(READINPUT_CGI)
        cgiFile.close()

        portnum = self.startServer(cgiFilename)
        d = client.getPage("http://localhost:%d/cgi" % portnum)
        d.addCallback(self._testReadEmptyInput_1)
        return d 

def update_bans(self):
        self.new_bans = NetworkDict()
        defers = []
        for url, url_filter in self.urls:
            defers.append(getPage(url.encode('utf8')).addCallback(self.got_bans,
                                                                  url_filter))
        DeferredList(defers).addCallback(self.bans_finished) 

def attemptRequestWithMaliciousURI(self, uri):
        """
        Attempt a request with the provided URI.

        @param uri: see L{URIInjectionTestsMixin}
        """
        client.getPage(uri) 

def testPayload(self):
        s = "0123456789" * 10
        return client.getPage(self.getURL("payload"), postdata=s
            ).addCallback(self.assertEquals, s
            ) 

def _requestTest(self, child, **kwargs):
        """
        Set up a resource on a distrib site using L{ResourcePublisher} and
        then retrieve it from a L{ResourceSubscription} via an HTTP client.

        @param child: The resource to publish using distrib.
        @param **kwargs: Extra keyword arguments to pass to L{getPage} when
            requesting the resource.

        @return: A L{Deferred} which fires with the result of the request.
        """
        distribRoot = resource.Resource()
        distribRoot.putChild("child", child)
        distribSite = server.Site(distribRoot)
        self.f1 = distribFactory = PBServerFactory(
            distrib.ResourcePublisher(distribSite))
        distribPort = reactor.listenTCP(
            0, distribFactory, interface="127.0.0.1")
        self.addCleanup(distribPort.stopListening)
        addr = distribPort.getHost()

        self.sub = mainRoot = distrib.ResourceSubscription(
            addr.host, addr.port)
        mainSite = server.Site(mainRoot)
        mainPort = reactor.listenTCP(0, mainSite, interface="127.0.0.1")
        self.addCleanup(mainPort.stopListening)
        mainAddr = mainPort.getHost()

        return client.getPage("http://%s:%s/child" % (
            mainAddr.host, mainAddr.port), **kwargs) 

def callRemote(self, method, *args, **kwargs):
        payload = SOAPpy.buildSOAP(args=args, kw=kwargs, method=method,
                                   header=self.header, namespace=self.namespace)
        return client.getPage(self.url, postdata=payload, method="POST",
                              headers={'content-type': 'text/xml',
                                       'SOAPAction': method}
                              ).addCallback(self._cbGotResult) 

def call(self, method, **kwargs):
        """
        Call the given remote method with the given arguments, as keywords.
        
        Returns a deferred, called with SOAPpy structure representing
        the soap response.
        
        @param method: The method name to call, eg. 'GetExternalIP'
        @param kwargs: The parameters of the call, as keywords
        @return: A deferred called with the external ip address of this host
        @rtype: L{twisted.internet.defer.Deferred}
        """
        payload = SOAPpy.buildSOAP(method=method, config=Config, namespace=self._prefix, kw=kwargs)
        # Here begins the nasty hack
        payload = payload.replace(
            # Upnp wants s: instead of SOAP-ENV
            'SOAP-ENV','s').replace(
            # Doesn't seem to like these encoding stuff
            'xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding/"', '').replace(
            'SOAP-ENC:root="1"', '').replace(
            # And it wants u: instead of ns1 namespace for arguments..
            'ns1','u')
        
        logging.debug("SOAP Payload:\n%s", payload)
        
        return client.getPage(self._url, postdata=payload, method="POST",
            headers={'content-type': 'text/xml',        'SOAPACTION': '%s#%s' % (self._prefix, method)}
    ).addCallbacks(self._got_page, self._got_error) 

def checkHTTPError(failure, url):
    failure.trap(weberror.Error)

    if failure.value.status=='401':
        print('>>>', sys.stderr, failure.getErrorMessage())

        username = input('用户名：')
        password = getpass.getpass('密码：')
        basicAuth = base64.encodestring('%s:%s' % (username, password))
        authHeader = 'Basic' + basicAuth.strip()
        # 加入验证信息后，尝试再次获取页面，
        return client.getPage(url.encode(), headers={'Authorization':authHeader})
    else:
        return failure 

def _get_changes(self):
        url = self._make_url()
        log.msg("GoogleCodeAtomPoller polling %s" % url)

        return getPage(url, timeout=self.pollinterval) 

def publicGetPage(*args, **kwargs):
        d = getPage(*args, **kwargs)
        return PublicDeferred(d) 

def start(self, data=None, std_alone=True):
        d = defer.succeed(self.printStatus())
        for feed in data:
        
            # Now we start telling the reactor that it has
            # to get all the feeds one by one...
            cached = self.isCached(feed)
            if not cached: 
                # When the feed is not cached, it's time to
                # go and get it from the web directly
                d.addCallback(self.getPage, feed[0])
                d.addErrback(self.gotError, (feed[0], 'getting'))
                
                # Parse the feed and if there's some errors call self.gotError
                d.addCallback(self.parseFeed)
                d.addErrback(self.gotError, (feed[0], 'parsing'))
                
                # Now memoize it, if there's some error call self.getError
                d.addCallback(self.memoize, feed[0])
                d.addErrback(self.gotError, (feed[0], 'memoizing'))
                
            else: # If it's cached
                d.addCallback(self.getPageFromMemory, feed[0])
                d.addErrback(self.gotError, (feed[0], 'getting from memory'))
            
            # When you get the raw structure you can work on it
            # to format in the best way you can think of.
            # For any error call self.gotError.
            d.addCallback(self.workOnPage, feed[0])
            d.addErrback(self.gotError, (feed[0], 'working on page'))
            
            # And when the for loop is ended we put 
            # stopWorking on the callback for the last 
            # feed gathered
            # This is only for testing purposes
            if std_alone:
                d.addCallback(self.stopWorking)
                d.addErrback(self.gotError, (feed[0], 'while stopping'))
        if not std_alone:
            return d 

def getPage(self, data, args):
        return client.getPage(args,timeout=TIMEOUT) 

def callRemote(self, method, *args, **kwargs):
        payload = SOAPpy.buildSOAP(args=args, kw=kwargs, method=method,
                                   header=self.header, namespace=self.namespace)
        return client.getPage(self.url, postdata=payload, method="POST",
                              headers={'content-type': 'text/xml',
                                       'SOAPAction': method}
                              ).addCallback(self._cbGotResult)