Python twisted.web.client.readBody() Examples

def test_ReadInput(self):
        cgiFilename = os.path.abspath(self.mktemp())
        with open(cgiFilename, 'wt') as cgiFile:
            cgiFile.write(READINPUT_CGI)

        portnum = self.startServer(cgiFilename)
        agent = client.Agent(reactor)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")
        d = agent.request(
            uri=url,
            method=b"POST",
            bodyProducer=client.FileBodyProducer(
                BytesIO(b"Here is your stdin")),
        )
        d.addCallback(client.readBody)
        d.addCallback(self._test_ReadInput_1)
        return d 

def test_create_generates_different_dataset_ids(self):
        """
        If two ``POST`` requests are made to create two different datasets,
        each dataset created is assigned a distinct ``dataset_id``.
        """
        creating = gatherResults([
            self.assertResponseCode(
                b"POST", b"/configuration/datasets",
                {u"primary": self.NODE_A},
                CREATED
            ).addCallback(readBody).addCallback(loads),
            self.assertResponseCode(
                b"POST", b"/configuration/datasets",
                {u"primary": self.NODE_A},
                CREATED
            ).addCallback(readBody).addCallback(loads),
        ])

        def created(datasets):
            first = datasets[0]
            second = datasets[1]
            self.assertNotEqual(first[u"dataset_id"], second[u"dataset_id"])
        creating.addCallback(created)
        return creating 

def verify(self, response):
        """
        Check the given response against the requirements defined by this
        instance.

        @param response: The response to check.
        @type response: L{twisted.web.iweb.IResponse}

        @return: A L{Deferred} that fires with C{None} after the response has
            been found to satisfy all the requirements or that fires with a
            L{Failure} if any part of the response is incorrect.
        """
        reading = readBody(response)
        reading.addCallback(self.decode)
        reading.addCallback(self._verifyWithBody, response)
        return reading 

def extractSuccessfulJSONResult(response):
    """
    Extract a successful API result from a HTTP response.

    @param response: The response to check.
    @type response: L{twisted.web.iweb.IResponse}

    @return: L{Deferred} that fires with the result part of the decoded JSON.

    @raises L{AssertionError}: If the response is not a successful one.
    """
    result = readBody(response)
    result.addCallback(loads)

    def getResult(data):
        if response.code > 299:
            raise AssertionError((response.code, data))
        return data
    result.addCallback(getResult)
    return result 

def testDistrib(self):
        # site1 is the publisher
        r1 = resource.Resource()
        r1.putChild(b"there", static.Data(b"root", "text/plain"))
        site1 = server.Site(r1)
        self.f1 = PBServerFactory(distrib.ResourcePublisher(site1))
        self.port1 = reactor.listenTCP(0, self.f1)
        self.sub = distrib.ResourceSubscription("127.0.0.1",
                                                self.port1.getHost().port)
        r2 = resource.Resource()
        r2.putChild(b"here", self.sub)
        f2 = MySite(r2)
        self.port2 = reactor.listenTCP(0, f2)
        agent = client.Agent(reactor)
        url = "http://127.0.0.1:{}/here/there".format(
            self.port2.getHost().port)
        url = url.encode("ascii")
        d = agent.request(b"GET", url)
        d.addCallback(client.readBody)
        d.addCallback(self.assertEqual, b'root')
        return d 

def twisted_coroutine_fetch(self, url, runner):
        body = [None]

        @gen.coroutine
        def f():
            # This is simpler than the non-coroutine version, but it cheats
            # by reading the body in one blob instead of streaming it with
            # a Protocol.
            client = Agent(self.reactor)
            response = yield client.request(b'GET', utf8(url))
            with warnings.catch_warnings():
                # readBody has a buggy DeprecationWarning in Twisted 15.0:
                # https://twistedmatrix.com/trac/changeset/43379
                warnings.simplefilter('ignore', category=DeprecationWarning)
                body[0] = yield readBody(response)
            self.stop_loop()
        self.io_loop.add_callback(f)
        runner()
        return body[0] 

def test_withPotentialDataLoss(self):
        """
        If the full body of the L{IResponse} passed to L{client.readBody} is
        not definitely received, the L{Deferred} returned by L{client.readBody}
        fires with a L{Failure} wrapping L{client.PartialDownloadError} with
        the content that was received.
        """
        response = DummyResponse()
        d = client.readBody(response)
        response.protocol.dataReceived(b"first")
        response.protocol.dataReceived(b"second")
        response.protocol.connectionLost(Failure(PotentialDataLoss()))
        failure = self.failureResultOf(d)
        failure.trap(client.PartialDownloadError)
        self.assertEqual({
            "status": failure.value.status,
            "message": failure.value.message,
            "body": failure.value.response,
        }, {
            "status": b"200",
            "message": b"OK",
            "body": b"firstsecond",
        }) 

def test_deprecatedTransport(self):
        """
        Calling L{client.readBody} with a transport that does not implement
        L{twisted.internet.interfaces.ITCPTransport} produces a deprecation
        warning, but no exception when cancelling.
        """
        response = DummyResponse(transportFactory=StringTransport)
        response.transport.abortConnection = None
        d = self.assertWarns(
            DeprecationWarning,
            'Using readBody with a transport that does not have an '
            'abortConnection method',
            __file__,
            lambda: client.readBody(response))
        d.cancel()
        self.failureResultOf(d, defer.CancelledError) 

def test_noProxyPassthrough(self):
        """
        The CGI script is never called with the Proxy header passed through.
        """
        cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")

        agent = client.Agent(reactor)

        headers = http_headers.Headers({b"Proxy": [b"foo"],
                                        b"X-Innocent-Header": [b"bar"]})
        d = agent.request(b"GET", url, headers=headers)

        def checkResponse(response):
            headers = json.loads(response.decode("ascii"))
            self.assertEqual(
                set(headers.keys()),
                {"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"})

        d.addCallback(client.readBody)
        d.addCallback(checkResponse)
        return d 

def _requestTest(self, child, **kwargs):
        """
        Set up a resource on a distrib site using L{ResourcePublisher} and
        then retrieve it from a L{ResourceSubscription} via an HTTP client.

        @param child: The resource to publish using distrib.
        @param **kwargs: Extra keyword arguments to pass to L{Agent.request} when
            requesting the resource.

        @return: A L{Deferred} which fires with the result of the request.
        """
        mainPort, mainAddr = self._setupDistribServer(child)
        agent = client.Agent(reactor)
        url = "http://%s:%s/child" % (mainAddr.host, mainAddr.port)
        url = url.encode("ascii")
        d = agent.request(b"GET", url, **kwargs)
        d.addCallback(client.readBody)
        return d 

def test_ReadAllInput(self):
        cgiFilename = os.path.abspath(self.mktemp())
        with open(cgiFilename, 'wt') as cgiFile:
            cgiFile.write(READALLINPUT_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")
        d = client.Agent(reactor).request(
            uri=url,
            method=b"POST",
            bodyProducer=client.FileBodyProducer(
                BytesIO(b"Here is your stdin")),
        )
        d.addCallback(client.readBody)
        d.addCallback(self._test_ReadAllInput_1)
        return d 

def _pushSuccess(self, result, updateDeferred):
        """
        Processes a successful push request. If the request resulted in a status code
        that's not a success, consider it a failure

        :param result: The HTTP response.
        :type result: twisted.web.iweb.IResponse
        :param updateDeferred: The deferred to make either succeed or fail depending on
            the status code.
        :type updateDeferred: twisted.internet.defer.Deferred
        """
        if result.code >= 200 and result.code < 300:
            updateDeferred.callback(result)
        else:
            d = readBody(result)
            d.addCallback(self._failedPushBodyRead, updateDeferred=updateDeferred)
            d.addErrback(self._pushFailed, updateDeferred=updateDeferred) 

def get_json(self, uri):
        """Make a GET request to an endpoint returning JSON and parse result

        :param uri: The URI to make a GET request to.
        :type uri: unicode

        :return: A deferred containing JSON parsed into a Python object.
        :rtype: twisted.internet.defer.Deferred[dict[any, any]]
        """
        logger.debug("HTTP GET %s", uri)

        response = yield self.agent.request(
            b"GET",
            uri.encode("utf8"),
        )
        body = yield readBody(response)
        try:
            # json.loads doesn't allow bytes in Python 3.5
            json_body = json.loads(body.decode("UTF-8"))
        except Exception as e:
            logger.exception("Error parsing JSON from %s", uri)
            raise
        defer.returnValue(json_body) 

def test_reports_metrics(self):
        """
        ``MetricsResource`` serves the metrics from the provided registry.
        """
        c = Counter('cc', 'A counter', registry=self.registry)
        c.inc()

        root = Resource()
        root.putChild(b'metrics', MetricsResource(registry=self.registry))
        server = reactor.listenTCP(0, Site(root))
        self.addCleanup(server.stopListening)

        agent = Agent(reactor)
        port = server.getHost().port
        url = "http://localhost:{port}/metrics".format(port=port)
        d = agent.request(b"GET", url.encode("ascii"))

        d.addCallback(readBody)
        d.addCallback(self.assertEqual, generate_latest(self.registry))

        return d 

def twisted_coroutine_fetch(self, url, runner):
        body = [None]

        @gen.coroutine
        def f():
            # This is simpler than the non-coroutine version, but it cheats
            # by reading the body in one blob instead of streaming it with
            # a Protocol.
            client = Agent(self.reactor)
            response = yield client.request(b'GET', utf8(url))
            with warnings.catch_warnings():
                # readBody has a buggy DeprecationWarning in Twisted 15.0:
                # https://twistedmatrix.com/trac/changeset/43379
                warnings.simplefilter('ignore', category=DeprecationWarning)
                body[0] = yield readBody(response)
            self.stop_loop()
        self.io_loop.add_callback(f)
        runner()
        return body[0] 

def run(reactor, cfg, tor, service):
    config = await tor.get_config()
    socks = await tor.protocol.get_info('net/listeners/socks')
    socks = socks['net/listeners/socks']
    socks_host, socks_port = socks.split(':')

    args = service.split(':')
    onion = args[1]
    cookie = args[2].split('=')[1]

    auth = '%s %s' % (onion, cookie)
    if auth not in config.HidServAuth:
        config.HidServAuth.append(auth)
    await config.save()

    agent = tor.web_agent()
    url = 'http://{}'.format(onion).encode('ascii')
    print("retrieving: {}".format(url))
    res = await agent.request(b'GET', url)
    print('Response: "{} {}" with {} bytes'.format(res.code, res.phrase, res.length))
    data = await readBody(res)
    print(data.decode('utf8')) 

def checkResponse(self, response, exp_result):
        # TODO: convert to log messages
        """
        print 'Response version:', response.version
        print 'Response code:', response.code
        print 'Response phrase:', response.phrase
        print 'Response headers:'
        print pformat(list(response.headers.getAllRawHeaders()))
        """
        """
        LOG.debug("Response Body %s", str(response.version))
        LOG.debug("Response Body %s", str(response.code))
        LOG.debug("Response Body %s", str(response.phrase))
        LOG.debug("Response Body %s",
                  str(list(response.headers.getAllRawHeaders())))
        LOG.debug("Expected Results %s", str(exp_result))
        """

        d = readBody(response)
        d.addCallback(self.assertResponse, exp_result)
        return d 

def getResponse(self, response):
        # TODO: convert to log messages
        """
        print 'Response version:', response.version
        print 'Response code:', response.code
        print 'Response phrase:', response.phrase
        print 'Response headers:'
        print pformat(list(response.headers.getAllRawHeaders()))
        """
        """
        LOG.debug("Response Body %s", str(response.version))
        LOG.debug("Response Body %s", str(response.code))
        LOG.debug("Response Body %s", str(response.phrase))
        LOG.debug("Response Body %s",
                  str(list(response.headers.getAllRawHeaders())))
        LOG.debug("Expected Results %s", str(exp_result))
        """

        d = readBody(response)
        return d 

def _requestTest(self, child, **kwargs):
        """
        Set up a resource on a distrib site using L{ResourcePublisher} and
        then retrieve it from a L{ResourceSubscription} via an HTTP client.

        @param child: The resource to publish using distrib.
        @param **kwargs: Extra keyword arguments to pass to L{Agent.request} when
            requesting the resource.

        @return: A L{Deferred} which fires with the result of the request.
        """
        mainPort, mainAddr = self._setupDistribServer(child)
        agent = client.Agent(reactor)
        url = "http://%s:%s/child" % (mainAddr.host, mainAddr.port)
        d = agent.request(b"GET", url, **kwargs)
        d.addCallback(client.readBody)
        return d 

def twisted_coroutine_fetch(self, url, runner):
        body = [None]

        @gen.coroutine
        def f():
            # This is simpler than the non-coroutine version, but it cheats
            # by reading the body in one blob instead of streaming it with
            # a Protocol.
            client = Agent(self.reactor)
            response = yield client.request(b'GET', utf8(url))
            with warnings.catch_warnings():
                # readBody has a buggy DeprecationWarning in Twisted 15.0:
                # https://twistedmatrix.com/trac/changeset/43379
                warnings.simplefilter('ignore', category=DeprecationWarning)
                body[0] = yield readBody(response)
            self.stop_loop()
        self.io_loop.add_callback(f)
        runner()
        return body[0] 

def test_key_not_found_is_raised_if_key_search_responds_404(self):
        """
        Test if key search request comes back with a 404 response then
        KeyNotFound is raised, with corresponding error message.
        """
        km = self._key_manager(url=NICKSERVER_URI)
        client.readBody = mock.Mock(return_value=defer.succeed(None))
        km._nicknym._async_client_pinned.request = mock.Mock(
            return_value=defer.succeed(None))
        url = NICKSERVER_URI + '?address=' + INVALID_MAIL_ADDRESS

        d = km._nicknym._fetch_and_handle_404_from_nicknym(url)

        def check_key_not_found_is_raised_if_404(_):
            used_kwargs = km._nicknym._async_client_pinned.request.call_args[1]
            check_404_callback = used_kwargs['callback']
            fake_response = mock.Mock()
            fake_response.code = NOT_FOUND
            with self.assertRaisesRegexp(errors.KeyNotFound,
                                         '404: Key not found. Request: '
                                         '%s' % url.replace('?', '\?')):
                check_404_callback(fake_response)

        d.addCallback(check_key_not_found_is_raised_if_404)
        return d 

def _fetch_key_with_address(self, km, address, key):
        """
        :returns: a Deferred that will fire with the OpenPGPKey
        """
        data = json.dumps({'address': address, 'openpgp': key})

        client.readBody = mock.Mock(return_value=defer.succeed(data))

        # mock the fetcher so it returns the key for ADDRESS_2
        km._nicknym._async_client_pinned.request = mock.Mock(
            return_value=defer.succeed(None))
        km.ca_cert_path = 'cacertpath'
        # try to key get without fetching from server
        d_fail = km.get_key(address, fetch_remote=False)
        d = self.assertFailure(d_fail, errors.KeyNotFound)
        # try to get key fetching from server.
        d.addCallback(lambda _: km.get_key(address))
        return d 

def testDistrib(self):
        # site1 is the publisher
        r1 = resource.Resource()
        r1.putChild("there", static.Data("root", "text/plain"))
        site1 = server.Site(r1)
        self.f1 = PBServerFactory(distrib.ResourcePublisher(site1))
        self.port1 = reactor.listenTCP(0, self.f1)
        self.sub = distrib.ResourceSubscription("127.0.0.1",
                                                self.port1.getHost().port)
        r2 = resource.Resource()
        r2.putChild("here", self.sub)
        f2 = MySite(r2)
        self.port2 = reactor.listenTCP(0, f2)
        agent = client.Agent(reactor)
        d = agent.request(b"GET", "http://127.0.0.1:%d/here/there" % \
                          (self.port2.getHost().port,))
        d.addCallback(client.readBody)
        d.addCallback(self.assertEqual, 'root')
        return d 

def testReadInput(self):
        cgiFilename = os.path.abspath(self.mktemp())
        with open(cgiFilename, 'wt') as cgiFile:
            cgiFile.write(READINPUT_CGI)

        portnum = self.startServer(cgiFilename)
        agent = client.Agent(reactor)
        d = agent.request(
            uri="http://localhost:%d/cgi" % (portnum,),
            method=b"POST",
            bodyProducer=client.FileBodyProducer(
                BytesIO(b"Here is your stdin")),
        )
        d.addCallback(client.readBody)
        d.addCallback(self._testReadInput_1)
        return d 

def test_withPotentialDataLoss(self):
        """
        If the full body of the L{IResponse} passed to L{client.readBody} is
        not definitely received, the L{Deferred} returned by L{client.readBody}
        fires with a L{Failure} wrapping L{client.PartialDownloadError} with
        the content that was received.
        """
        response = DummyResponse()
        d = client.readBody(response)
        response.protocol.dataReceived(b"first")
        response.protocol.dataReceived(b"second")
        response.protocol.connectionLost(Failure(PotentialDataLoss()))
        failure = self.failureResultOf(d)
        failure.trap(client.PartialDownloadError)
        self.assertEqual({
            "status": failure.value.status,
            "message": failure.value.message,
            "body": failure.value.response,
        }, {
            "status": b"200",
            "message": b"OK",
            "body": b"firstsecond",
        }) 

def test_deprecatedTransport(self):
        """
        Calling L{client.readBody} with a transport that does not implement
        L{twisted.internet.interfaces.ITCPTransport} produces a deprecation
        warning, but no exception when cancelling.
        """
        response = DummyResponse(transportFactory=StringTransport)
        response.transport.abortConnection = None
        d = self.assertWarns(
            DeprecationWarning,
            'Using readBody with a transport that does not have an '
            'abortConnection method',
            __file__,
            lambda: client.readBody(response))
        d.cancel()
        self.failureResultOf(d, defer.CancelledError) 

def test_noProxyPassthrough(self):
        """
        The CGI script is never called with the Proxy header passed through.
        """
        cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)

        agent = client.Agent(reactor)

        headers = http_headers.Headers({"Proxy": ["foo"],
                                        "X-Innocent-Header": ["bar"]})
        d = agent.request(b"GET", url, headers=headers)

        def checkResponse(response):
            headers = json.loads(response)
            self.assertEqual(
                set(headers.keys()),
                {"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"})

        d.addCallback(client.readBody)
        d.addCallback(checkResponse)
        return d 

def cbRequest(self, response):
        if self.verbose:
            print 'Response version:', response.version
            print 'Response code:', response.code
            print 'Response phrase:', response.phrase
            print 'Response headers:'
            print pformat(list(response.headers.getAllRawHeaders()))
        d = self.reqQ.readBody(response)
        d.addCallback(self.cbBody)
        d.addErrback(self.onGetPlaylistError)
        return d 

def testReadAllInput(self):
        cgiFilename = os.path.abspath(self.mktemp())
        with open(cgiFilename, 'wt') as cgiFile:
            cgiFile.write(READALLINPUT_CGI)

        portnum = self.startServer(cgiFilename)
        d = client.Agent(reactor).request(
            uri="http://localhost:%d/cgi" % (portnum,),
            method=b"POST",
            bodyProducer=client.FileBodyProducer(
                BytesIO(b"Here is your stdin")),
        )
        d.addCallback(client.readBody)
        d.addCallback(self._testReadAllInput_1)
        return d 

def get_body(what, f):
    def cb(resp):
        d = readBody(resp)
        d.addCallback(partial(f, what))
        return d
    return cb