Python twisted.internet.defer.DeferredList() Examples

def testDeferredListWithAlreadyFiredDeferreds(self):
        # Create some deferreds, and err one, call the other
        d1 = defer.Deferred()
        d2 = defer.Deferred()
        d1.errback(GenericError('Bang'))
        d2.callback(2)

        # *Then* build the DeferredList
        dl = defer.DeferredList([d1, d2])

        result = []
        dl.addCallback(result.append)

        self.assertEqual(1, len(result))

        d1.addErrback(lambda e: None)  # Swallow error 

def test_errors(self):
        """
        Verify that for each way a method exposed via XML-RPC can fail, the
        correct 'Content-type' header is set in the response and that the
        client-side Deferred is errbacked with an appropriate C{Fault}
        instance.
        """
        dl = []
        for code, methodName in [(666, "fail"), (666, "deferFail"),
                                 (12, "fault"), (23, "noSuchMethod"),
                                 (17, "deferFault"), (42, "SESSION_TEST")]:
            d = self.proxy().callRemote(methodName)
            d = self.assertFailure(d, xmlrpc.Fault)
            d.addCallback(lambda exc, code=code:
                self.assertEqual(exc.faultCode, code))
            dl.append(d)
        d = defer.DeferredList(dl, fireOnOneErrback=True)
        def cb(ign):
            for factory in self.factories:
                self.assertEqual(factory.headers[b'content-type'],
                                  b'text/xml; charset=utf-8')
            self.flushLoggedErrors(TestRuntimeError, TestValueError)
        d.addCallback(cb)
        return d 

def tearDown(self):
        if self.agent:
            # clean up connections for twisted.web.client.Agent test.
            self.agent.closeCachedConnections()
            self.agent = None

        # If the test indicated it might leave some server-side connections
        # around, clean them up.
        connections = list(self.wrapper.protocols.keys())
        # If there are fewer server-side connections than requested,
        # that's okay.  Some might have noticed that the client closed
        # the connection and cleaned up after themselves.
        for n in range(min(len(connections), self.cleanupServerConnections)):
            proto = connections.pop()
            msg("Closing %r" % (proto,))
            proto.transport.abortConnection()
        d = self.port.stopListening()

        return defer.DeferredList([waitUntilAllDisconnected(
            reactor, list(self.wrapper.protocols.keys())), d]) 

def gather(self, futures, consume_exceptions=True):
        def completed(res):
            rtn = []
            for (ok, value) in res:
                rtn.append(value)
                if not ok and not consume_exceptions:
                    value.raiseException()
            return rtn

        # XXX if consume_exceptions is False in asyncio.gather(), it will
        # abort on the first raised exception -- should we set
        # fireOnOneErrback=True (if consume_exceptions=False?) -- but then
        # we'll have to wrap the errback() to extract the "real" failure
        # from the FirstError that gets thrown if you set that ...

        dl = DeferredList(list(futures), consumeErrors=consume_exceptions)
        # we unpack the (ok, value) tuples into just a list of values, so
        # that the callback() gets the same value in asyncio and Twisted.
        add_callbacks(dl, completed, None)
        return dl 

def testResults(self):
        inputOutput = [
            ("add", (2, 3), 5),
            ("defer", ("a",), "a"),
            ("dict", ({"a": 1}, "a"), 1),
            ("triple", ("a", 1), ["a", 1, None])]

        dl = []
        for meth, args, outp in inputOutput:
            d = self.proxy().callRemote(meth, *args)
            d.addCallback(self.assertEqual, outp)
            dl.append(d)

        # SOAPpy kinda blows.
        d = self.proxy().callRemote('complex')
        d.addCallback(lambda result: result._asdict())
        d.addCallback(self.assertEqual, {"a": ["b", "c", 12, []], "D": "foo"})
        dl.append(d)

        # We now return to our regularly scheduled program, already in progress.
        return defer.DeferredList(dl, fireOnOneErrback=True) 

def logOn(self, chatui):
        """
        @returns: this breaks with L{interfaces.IAccount}
        @returntype: DeferredList of L{interfaces.IClient}s
        """
        # Overriding basesupport's implementation on account of the
        # fact that _startLogOn tends to return a deferredList rather
        # than a simple Deferred, and we need to do registerAccountClient.
        if (not self._isConnecting) and (not self._isOnline):
            self._isConnecting = 1
            d = self._startLogOn(chatui)
            d.addErrback(self._loginFailed)
            def registerMany(results):
                for success, result in results:
                    if success:
                        chatui.registerAccountClient(result)
                        self._cb_logOn(result)
                    else:
                        log.err(result)
            d.addCallback(registerMany)
            return d
        else:
            raise error.ConnectionError("Connection in progress") 

def get(self):
        """HTTP Get

        Returns basic information about the version and how many clients are
        connected in a JSON object.

        """
        self._healthy = True
        self._health_checks = dict(
            version=__version__,
            clients=len(getattr(self.application, 'clients', ()))
        )

        dl = DeferredList([
            self._check_table(self.db.router.table),
            self._check_table(self.db.message.table, "storage"),
        ])
        dl.addBoth(self._finish_response) 

def _add_nodes_to_cluster(self, reactor, cluster, results):
        def add_node(node, index):
            # The control should be already fully configured.
            if node is not cluster.control_node:
                return self._add_node_to_cluster(reactor, cluster, node, index)

        for i, d in enumerate(results):
            d.addCallback(add_node, i)

        # Failure to add any one node to a cluster is not fatal,
        # we are happy with a partial success as long as we've
        # managed to configure the control node.
        # So, just wait until we are done with all nodes.
        d = DeferredList(results)
        d.addCallback(lambda _: cluster)
        return d 

def test_callFromThread(self):
        """
        Test callFromThread functionality: from the main thread, and from
        another thread.
        """
        def cb(ign):
            firedByReactorThread = defer.Deferred()
            firedByOtherThread = defer.Deferred()

            def threadedFunc():
                reactor.callFromThread(firedByOtherThread.callback, None)

            reactor.callInThread(threadedFunc)
            reactor.callFromThread(firedByReactorThread.callback, None)

            return defer.DeferredList(
                [firedByReactorThread, firedByOtherThread],
                fireOnOneErrback=True)
        return self._waitForThread().addCallback(cb) 

def testCooperation(self):
        L = []
        def myiter(things):
            for th in things:
                L.append(th)
                yield None

        groupsOfThings = ['abc', (1, 2, 3), 'def', (4, 5, 6)]

        c = task.Cooperator()
        tasks = []
        for stuff in groupsOfThings:
            tasks.append(c.coiterate(myiter(stuff)))

        return defer.DeferredList(tasks).addCallback(
            lambda ign: self.assertEqual(tuple(L), sum(zip(*groupsOfThings), ()))) 

def _cbGetFileSize(self, attrs, rf, lf):
        if not stat.S_ISREG(attrs['permissions']):
            rf.close()
            lf.close()
            return "Can't get non-regular file: %s" % rf.name
        rf.size = attrs['size']
        bufferSize = self.client.transport.conn.options['buffersize']
        numRequests = self.client.transport.conn.options['requests']
        rf.total = 0.0
        dList = []
        chunks = []
        startTime = self.reactor.seconds()
        for i in range(numRequests):
            d = self._cbGetRead('', rf, lf, chunks, 0, bufferSize, startTime)
            dList.append(d)
        dl = defer.DeferredList(dList, fireOnOneErrback=1)
        dl.addCallback(self._cbGetDone, rf, lf)
        return dl 

def test_cancelDeferredListWithException(self):
        """
        Cancelling a L{defer.DeferredList} will cancel every L{defer.Deferred}
        in the list even exceptions raised from the C{cancel} method of the
        L{defer.Deferred}s.
        """
        def cancellerRaisesException(deferred):
            """
            A L{defer.Deferred} canceller that raises an exception.

            @param deferred: The cancelled L{defer.Deferred}.
            """
            raise RuntimeError("test")
        deferredOne = defer.Deferred(cancellerRaisesException)
        deferredTwo = defer.Deferred()
        deferredList = defer.DeferredList([deferredOne, deferredTwo])
        deferredList.cancel()
        self.failureResultOf(deferredTwo, defer.CancelledError)
        errors = self.flushLoggedErrors(RuntimeError)
        self.assertEqual(len(errors), 1) 

def test_failedCertificateVerification(self):
        """
        Check that connecting with a certificate not accepted by the server CA
        fails.
        """
        onServerLost = defer.Deferred()
        onClientLost = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey,
                            certificate=self.sCert, verify=False,
                            requireCertificate=False),
                      sslverify.OpenSSLCertificateOptions(verify=True,
                            requireCertificate=False, caCerts=[self.cCert]),
                      onServerLost=onServerLost,
                      onClientLost=onClientLost)

        d = defer.DeferredList([onClientLost, onServerLost],
                               consumeErrors=True)
        def afterLost(result):
            ((cSuccess, cResult), (sSuccess, sResult)) = result
            self.assertFalse(cSuccess)
            self.assertFalse(sSuccess)

        return d.addCallback(afterLost) 

def test_cancelDeferredListWithFireOnOneErrbackAllDeferredsCallback(self):
        """
        When cancelling an unfired L{defer.DeferredList} with the flag
        C{fireOnOneErrback} set, if all the L{defer.Deferred} callbacks
        in its canceller, the L{defer.DeferredList} will callback with a
        C{list} of (success, result) C{tuple}s.
        """
        deferredOne = defer.Deferred(fakeCallbackCanceller)
        deferredTwo = defer.Deferred(fakeCallbackCanceller)
        deferredList = defer.DeferredList([deferredOne, deferredTwo],
                                          fireOnOneErrback=True)
        deferredList.cancel()
        result = self.successResultOf(deferredList)
        self.assertTrue(result[0][0])
        self.assertEqual(result[0][1], "Callback Result")
        self.assertTrue(result[1][0])
        self.assertEqual(result[1][1], "Callback Result") 

def deferredDict(d):
    """
    Just like a :class:`defer.DeferredList` but instead accepts and returns a :class:`dict`.

    Args:
        d: A :class:`dict` whose values are all :class:`defer.Deferred` objects.

    Returns:
        :class:`defer.DeferredList` whose callback will be given a dictionary whose
        keys are the same as the parameter :obj:`d` and whose values are the results
        of each individual deferred call.
    """
    if len(d) == 0:
        return defer.succeed({})

    def handle(results, names):
        rvalue = {}
        for index in range(len(results)):
            rvalue[names[index]] = results[index][1]
        return rvalue

    dl = defer.DeferredList(d.values())
    return dl.addCallback(handle, d.keys()) 

def test_cancelDeferredListCallback(self):
        """
        When cancelling an unfired L{defer.DeferredList} without the
        C{fireOnOneCallback} and C{fireOnOneErrback} flags set, the
        L{defer.DeferredList} will be callback with a C{list} of
        (success, result) C{tuple}s.
        """
        deferredOne = defer.Deferred(fakeCallbackCanceller)
        deferredTwo = defer.Deferred()
        deferredList = defer.DeferredList([deferredOne, deferredTwo])
        deferredList.cancel()
        self.failureResultOf(deferredTwo, defer.CancelledError)
        result = self.successResultOf(deferredList)
        self.assertTrue(result[0][0])
        self.assertEqual(result[0][1], "Callback Result")
        self.assertFalse(result[1][0])
        self.assertTrue(result[1][1].check(defer.CancelledError)) 

def test_cancelDeferredListWithFireOnOneErrback(self):
        """
        When cancelling an unfired L{defer.DeferredList} with the flag
        C{fireOnOneErrback} set, cancel every L{defer.Deferred} in the list.
        """
        deferredOne = defer.Deferred()
        deferredTwo = defer.Deferred()
        deferredList = defer.DeferredList([deferredOne, deferredTwo],
                                          fireOnOneErrback=True)
        deferredList.cancel()
        self.failureResultOf(deferredOne, defer.CancelledError)
        self.failureResultOf(deferredTwo, defer.CancelledError)
        deferredListFailure = self.failureResultOf(deferredList,
                                                   defer.FirstError)
        firstError = deferredListFailure.value
        self.assertTrue(firstError.subFailure.check(defer.CancelledError)) 

def set(self, keyword, key, value, ttl=604800):
        """
        Set the given key/value tuple at the hash of the given keyword.
        All values stored in the DHT are stored as dictionaries of key/value
        pairs. If a value already exists for a given keyword, the new key/value
        pair will be appended to the dictionary.

        Args:
            keyword: The keyword to use. Should be hashed with hash160 before
                passing it in here.
            key: the 20 byte hash of the data.
            value: a serialized `protos.objects.Node` object which serves as a
                pointer to the node storing the data.

        Return: True if at least one peer responded. False if the store rpc
            completely failed.
        """
        if len(keyword) != 20:
            return defer.succeed(False)

        self.log.debug("setting '%s' on network" % keyword.encode("hex"))

        def store(nodes):
            self.log.debug("setting '%s' on %s" % (keyword.encode("hex"), [str(i) for i in nodes]))
            ds = [self.protocol.callStore(node, keyword, key, value, ttl) for node in nodes]

            keynode = Node(keyword)
            if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
                self.storage[keyword] = (key, value, ttl)
                self.log.debug("got a store request from %s, storing value" % str(self.node))

            return defer.DeferredList(ds).addCallback(_anyRespondSuccess)

        node = Node(keyword)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("there are no known neighbors to set keyword %s" % keyword.encode("hex"))
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(store) 

def testFailedVerify(self):
        org = "twisted.test.test_ssl"
        self.setupServerAndClient(
            (org, org + ", client"), {},
            (org, org + ", server"), {})

        def verify(*a):
            return False
        self.clientCtxFactory.getContext().set_verify(SSL.VERIFY_PEER, verify)

        serverConnLost = defer.Deferred()
        serverProtocol = protocol.Protocol()
        serverProtocol.connectionLost = serverConnLost.callback
        serverProtocolFactory = protocol.ServerFactory()
        serverProtocolFactory.protocol = lambda: serverProtocol
        self.serverPort = serverPort = reactor.listenSSL(0,
            serverProtocolFactory, self.serverCtxFactory)

        clientConnLost = defer.Deferred()
        clientProtocol = protocol.Protocol()
        clientProtocol.connectionLost = clientConnLost.callback
        clientProtocolFactory = protocol.ClientFactory()
        clientProtocolFactory.protocol = lambda: clientProtocol
        reactor.connectSSL('127.0.0.1',
            serverPort.getHost().port, clientProtocolFactory, self.clientCtxFactory)

        dl = defer.DeferredList([serverConnLost, clientConnLost], consumeErrors=True)
        return dl.addCallback(self._cbLostConns) 

def process_item(self, item, spider):
        info = self.spiderinfo
        requests = arg_to_iter(self.get_media_requests(item, info))
        dlist = [self._process_request(r, info, item, spider) for r in requests]
        dfd = DeferredList(dlist, consumeErrors=1)
        return dfd.addCallback(self.item_completed, item, info) 

def testDeferredListEmpty(self):
        """Testing empty DeferredList."""
        dl = defer.DeferredList([])
        dl.addCallback(self.cb_empty) 

def test_cancelFiredOnOneErrbackDeferredList(self):
        """
        When a L{defer.DeferredList} has fired because one L{defer.Deferred} in
        the list fired with a failure result, the cancellation will do
        nothing instead of cancelling the rest of the L{defer.Deferred}s.
        """
        deferredOne = defer.Deferred()
        deferredTwo = defer.Deferred()
        deferredList = defer.DeferredList([deferredOne, deferredTwo],
                                          fireOnOneErrback=True)
        deferredOne.errback(GenericError("test"))
        deferredList.cancel()
        self.assertNoResult(deferredTwo)
        self.failureResultOf(deferredOne, GenericError)
        self.failureResultOf(deferredList, defer.FirstError) 

def gotClient(client):
    d1 = client.ping().addCallback(gotPing).addErrback(gotTransportError)

    d2 = client.add(1, 2).addCallback(gotAddResults).addErrback(gotTransportError)

    w = Work(num1=2, num2=3, op=Operation.ADD)

    d3 = client.calculate(1, w).addCallbacks(gotCalculateResults,
        gotCalculateErrors).addErrback(gotTransportError)

    w = Work(num1=2, num2=3, op=Operation.SUBTRACT)

    d4 = client.calculate(2, w).addCallbacks(gotCalculateResults,
        gotCalculateErrors).addErrback(gotTransportError)

    w = Work(num1=2, num2=3, op=Operation.MULTIPLY)

    d5 = client.calculate(3, w).addCallbacks(gotCalculateResults,
        gotCalculateErrors).addErrback(gotTransportError)

    w = Work(num1=2, num2=3, op=Operation.DIVIDE)

    d6 = client.calculate(4, w).addCallbacks(gotCalculateResults,
        gotCalculateErrors).addErrback(gotTransportError)

    # This will fire an errback
    w = Work(num1=2, num2=0, op=Operation.DIVIDE)

    d7 = client.calculate(5, w).addCallbacks(gotCalculateResults,
        gotCalculateErrors).addErrback(gotTransportError)

    d8 = client.zip()

    dl = defer.DeferredList([d1, d2, d3, d4, d5, d6, d7, d8])

    dl.addCallback(lambda _: reactor.stop()) 

def test_cancelDeferredListWithOriginalDeferreds(self):
        """
        Cancelling a L{defer.DeferredList} will cancel the original
        L{defer.Deferred}s passed in.
        """
        deferredOne = defer.Deferred()
        deferredTwo = defer.Deferred()
        argumentList = [deferredOne, deferredTwo]
        deferredList = defer.DeferredList(argumentList)
        deferredThree = defer.Deferred()
        argumentList.append(deferredThree)
        deferredList.cancel()
        self.failureResultOf(deferredOne, defer.CancelledError)
        self.failureResultOf(deferredTwo, defer.CancelledError)
        self.assertNoResult(deferredThree) 

def stop(self):
        """
        Stops simultaneously all the crawling jobs taking place.

        Returns a deferred that is fired when they all have ended.
        """
        return defer.DeferredList([c.stop() for c in list(self.crawlers)]) 

def test_cancelDeferredListWithFireOnOneCallbackAndDeferredCallback(self):
        """
        When cancelling an unfired L{defer.DeferredList} with the flag
        C{fireOnOneCallback} set, if one of the L{defer.Deferred} callbacks
        in its canceller, the L{defer.DeferredList} will callback with the
        result and the index of the L{defer.Deferred} in a C{tuple}.
        """
        deferredOne = defer.Deferred(fakeCallbackCanceller)
        deferredTwo = defer.Deferred()
        deferredList = defer.DeferredList([deferredOne, deferredTwo],
                                          fireOnOneCallback=True)
        deferredList.cancel()
        self.failureResultOf(deferredTwo, defer.CancelledError)
        result = self.successResultOf(deferredList)
        self.assertEqual(result, ("Callback Result", 0)) 

def join(self):
        """
        join()

        Returns a deferred that is fired when all managed :attr:`crawlers` have
        completed their executions.
        """
        while self._active:
            yield defer.DeferredList(self._active) 

def test_cancelDeferredList(self):
        """
        When cancelling an unfired L{defer.DeferredList}, cancel every
        L{defer.Deferred} in the list.
        """
        deferredOne = defer.Deferred()
        deferredTwo = defer.Deferred()
        deferredList = defer.DeferredList([deferredOne, deferredTwo])
        deferredList.cancel()
        self.failureResultOf(deferredOne, defer.CancelledError)
        self.failureResultOf(deferredTwo, defer.CancelledError) 

def parallel(iterable, count, callable, *args, **named):
    """Execute a callable over the objects in the given iterable, in parallel,
    using no more than ``count`` concurrent calls.

    Taken from: https://jcalderone.livejournal.com/24285.html
    """
    coop = task.Cooperator()
    work = (callable(elem, *args, **named) for elem in iterable)
    return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) 

def _runTest(self, num_threads, multiple_services, service_maker, details, num_requests, do_auth):

        if multiple_services:
            services = [service_maker() for _ in range(num_threads)]
        else:
            services = [service_maker()] * num_threads

        # Warm up each service before starting timer
        for n, svc in enumerate(services):
            record = yield svc.recordWithShortName(RecordType.user, details["user"].format(n + 1))
            self.assertTrue(isinstance(record, DirectoryRecord))

        start = time.time()
        ctr = [0]

        @inlineCallbacks
        def _records(n):
            for _ in range(num_requests):
                record = yield services[n].recordWithShortName(RecordType.user, details["user"].format(n + 1))
                self.assertTrue(isinstance(record, DirectoryRecord))
                ctr[0] += 1

        @inlineCallbacks
        def _auth(n):
            record = yield services[n].recordWithShortName(RecordType.user, details["user"].format(n + 1))
            for _ in range(num_requests):
                result = yield record.verifyPlaintextPassword(details["pswd"].format(n + 1))
                self.assertTrue(result)
                ctr[0] += 1

        dl = []
        for i in range(num_threads):
            dl.append((_auth if do_auth else _records)(i))

        dl = DeferredList(dl)
        yield dl

        returnValue((time.time() - start, ctr[0],))