Python twisted.internet.defer.DeferredSemaphore() Examples

def __init__(self, name, sygnal, config, canonical_reg_id_store):
        super(GcmPushkin, self).__init__(name, sygnal, config)

        nonunderstood = set(self.cfg.keys()).difference(self.UNDERSTOOD_CONFIG_FIELDS)
        if len(nonunderstood) > 0:
            logger.warning(
                "The following configuration fields are not understood: %s",
                nonunderstood,
            )

        self.http_pool = HTTPConnectionPool(reactor=sygnal.reactor)
        self.max_connections = self.get_config(
            "max_connections", DEFAULT_MAX_CONNECTIONS
        )
        self.connection_semaphore = DeferredSemaphore(self.max_connections)
        self.http_pool.maxPersistentPerHost = self.max_connections

        tls_client_options_factory = ClientTLSOptionsFactory()

        self.http_agent = Agent(
            reactor=sygnal.reactor,
            pool=self.http_pool,
            contextFactory=tls_client_options_factory,
        )

        self.db = sygnal.database
        self.canonical_reg_id_store = canonical_reg_id_store

        self.api_key = self.get_config("api_key")
        if not self.api_key:
            raise PushkinSetupException("No API key set in config") 

def test_make_database_unpool_creates_unpool(self):
        pool = threads.make_database_unpool()
        self.assertThat(pool, IsInstance(ThreadUnpool))
        self.assertThat(pool.contextFactory, Is(orm.ExclusivelyConnected))
        self.assertThat(pool.lock, IsInstance(DeferredSemaphore))
        self.assertThat(
            pool.lock.limit, Equals(threads.max_threads_for_database_pool)
        ) 

def make_database_unpool(maxthreads=max_threads_for_database_pool):
    """Create a general non-thread-pool for database activity.

    Its consumer are the old-school web application, i.e. the plain HTTP and
    HTTP API services, and the WebSocket service, for the responsive web UI.
    Each thread is fully connected to the database.

    However, this is a :class:`ThreadUnpool`, which means that threads are not
    actually pooled: a new thread is created for each task. This is ideal for
    testing, to improve isolation between tests.
    """
    return ThreadUnpool(DeferredSemaphore(maxthreads), ExclusivelyConnected) 

def run(self, concurrency=1):
        """Ask the rack controllers to download the region's boot resources.

        Report the results via the log.

        :param concurrency: Limit the number of rack controllers importing at
            one time to no more than `concurrency`.
        """
        lock = DeferredSemaphore(concurrency)

        def report(results):
            message_success = (
                "Rack controller (%s) has imported boot resources."
            )
            message_failure = (
                "Rack controller (%s) failed to import boot resources."
            )
            message_disconn = (
                "Rack controller (%s) did not import boot resources; it is "
                "not connected to the region at this time."
            )
            for system_id, (success, result) in zip(self.system_ids, results):
                if success:
                    log.msg(message_success % system_id)
                elif result.check(NoConnectionsAvailable):
                    log.msg(message_disconn % system_id)
                else:
                    log.err(result, message_failure % system_id)

        return (
            self(lock)
            .addCallback(report)
            .addErrback(log.err, "General failure syncing boot resources.")
        ) 

def _isThreadpoolQuiet(self, pool):
        """Is the given threadpool quiet, i.e. not in use?

        This can handle MAAS's custom threadpools as well as Twisted's default
        implementation.
        """
        lock = getattr(pool, "lock", None)
        if isinstance(lock, defer.DeferredLock):
            return not lock.locked
        elif isinstance(lock, defer.DeferredSemaphore):
            return lock.tokens == lock.limit
        else:
            return len(pool.working) == 0 

def testSemaphore(self):
        N = 13
        sem = defer.DeferredSemaphore(N)

        controlDeferred = defer.Deferred()
        def helper(self, arg):
            self.arg = arg
            return controlDeferred

        results = []
        uniqueObject = object()
        resultDeferred = sem.run(helper, self=self, arg=uniqueObject)
        resultDeferred.addCallback(results.append)
        resultDeferred.addCallback(self._incr)
        self.assertEquals(results, [])
        self.assertEquals(self.arg, uniqueObject)
        controlDeferred.callback(None)
        self.assertEquals(results.pop(), None)
        self.assertEquals(self.counter, 1)

        self.counter = 0
        for i in range(1, 1 + N):
            sem.acquire().addCallback(self._incr)
            self.assertEquals(self.counter, i)

        sem.acquire().addCallback(self._incr)
        self.assertEquals(self.counter, N)

        sem.release()
        self.assertEquals(self.counter, N + 1)

        for i in range(1, 1 + N):
            sem.release()
            self.assertEquals(self.counter, N + 1) 

def test_cancelSemaphoreBeforeAcquired(self):
        """
        When canceling a L{Deferred} from a L{DeferredSemaphore} that does
        not yet have the semaphore (i.e., the L{Deferred} has not fired),
        the cancel should cause a L{defer.CancelledError} failure.
        """
        sem = defer.DeferredSemaphore(1)
        sem.acquire()
        d = sem.acquire()
        self.assertFailure(d, defer.CancelledError)
        d.cancel()
        return d 

def test_cancelSemaphoreAfterAcquired(self):
        """
        When canceling a L{Deferred} from a L{DeferredSemaphore} that
        already has the semaphore, the cancel should have no effect.
        """
        def _failOnErrback(_):
            self.fail("Unexpected errback call!")

        sem = defer.DeferredSemaphore(1)
        d = sem.acquire()
        d.addErrback(_failOnErrback)
        d.cancel() 

def test_semaphoreInvalidTokens(self):
        """
        If the token count passed to L{DeferredSemaphore} is less than one
        then L{ValueError} is raised.
        """
        self.assertRaises(ValueError, defer.DeferredSemaphore, 0)
        self.assertRaises(ValueError, defer.DeferredSemaphore, -1) 

def runScript(self, commands):
        """
        Run each command in sequence and return a Deferred that fires when all
        commands are completed.

        @param commands: A list of strings containing sftp commands.

        @return: A C{Deferred} that fires when all commands are completed. The
        payload is a list of response strings from the server, in the same
        order as the commands.
        """
        sem = defer.DeferredSemaphore(1)
        dl = [sem.run(self.runCommand, command) for command in commands]
        return defer.gatherResults(dl) 

def runScript(self, commands):
        """
        Run each command in sequence and return a Deferred that fires when all
        commands are completed.

        @param commands: A list of strings containing sftp commands.

        @return: A C{Deferred} that fires when all commands are completed. The
        payload is a list of response strings from the server, in the same
        order as the commands.
        """
        sem = defer.DeferredSemaphore(1)
        dl = [sem.run(self.runCommand, command) for command in commands]
        return defer.gatherResults(dl) 

def test_cancelSemaphoreBeforeAcquired(self):
        """
        When canceling a L{Deferred} from a L{DeferredSemaphore} that does
        not yet have the semaphore (i.e., the L{Deferred} has not fired),
        the cancel should cause a L{defer.CancelledError} failure.
        """
        sem = defer.DeferredSemaphore(1)
        sem.acquire()
        d = sem.acquire()
        d.cancel()
        self.assertImmediateFailure(d, defer.CancelledError) 

def test_cancelSemaphoreAfterAcquired(self):
        """
        When canceling a L{Deferred} from a L{DeferredSemaphore} that
        already has the semaphore, the cancel should have no effect.
        """
        def _failOnErrback(_):
            self.fail("Unexpected errback call!")

        sem = defer.DeferredSemaphore(1)
        d = sem.acquire()
        d.addErrback(_failOnErrback)
        d.cancel() 

def test_semaphoreInvalidTokens(self):
        """
        If the token count passed to L{DeferredSemaphore} is less than one
        then L{ValueError} is raised.
        """
        self.assertRaises(ValueError, defer.DeferredSemaphore, 0)
        self.assertRaises(ValueError, defer.DeferredSemaphore, -1) 

def runScript(self, commands):
        """
        Run each command in sequence and return a Deferred that fires when all
        commands are completed.

        @param commands: A list of strings containing sftp commands.

        @return: A C{Deferred} that fires when all commands are completed. The
        payload is a list of response strings from the server, in the same
        order as the commands.
        """
        sem = defer.DeferredSemaphore(1)
        dl = [sem.run(self.runCommand, command) for command in commands]
        return defer.gatherResults(dl) 

def test_cancelSemaphoreBeforeAcquired(self):
        """
        When canceling a L{Deferred} from a L{DeferredSemaphore} that does
        not yet have the semaphore (i.e., the L{Deferred} has not fired),
        the cancel should cause a L{defer.CancelledError} failure.
        """
        sem = defer.DeferredSemaphore(1)
        sem.acquire()
        d = sem.acquire()
        d.cancel()
        self.assertImmediateFailure(d, defer.CancelledError) 

def test_cancelSemaphoreAfterAcquired(self):
        """
        When canceling a L{Deferred} from a L{DeferredSemaphore} that
        already has the semaphore, the cancel should have no effect.
        """
        def _failOnErrback(_):
            self.fail("Unexpected errback call!")

        sem = defer.DeferredSemaphore(1)
        d = sem.acquire()
        d.addErrback(_failOnErrback)
        d.cancel() 

def test_semaphoreInvalidTokens(self):
        """
        If the token count passed to L{DeferredSemaphore} is less than one
        then L{ValueError} is raised.
        """
        self.assertRaises(ValueError, defer.DeferredSemaphore, 0)
        self.assertRaises(ValueError, defer.DeferredSemaphore, -1) 

def testSemaphore(self):
        N = 13
        sem = defer.DeferredSemaphore(N)

        controlDeferred = defer.Deferred()
        def helper(self, arg):
            self.arg = arg
            return controlDeferred

        results = []
        uniqueObject = object()
        resultDeferred = sem.run(helper, self=self, arg=uniqueObject)
        resultDeferred.addCallback(results.append)
        resultDeferred.addCallback(self._incr)
        self.assertEquals(results, [])
        self.assertEquals(self.arg, uniqueObject)
        controlDeferred.callback(None)
        self.assertEquals(results.pop(), None)
        self.assertEquals(self.counter, 1)

        self.counter = 0
        for i in range(1, 1 + N):
            sem.acquire().addCallback(self._incr)
            self.assertEquals(self.counter, i)


        success = []
        def fail(r):
            success.append(False)
        def succeed(r):
            success.append(True)
        d = sem.acquire().addCallbacks(fail, succeed)
        d.cancel()
        self.assertEquals(success, [True])

        sem.acquire().addCallback(self._incr)
        self.assertEquals(self.counter, N)

        sem.release()
        self.assertEquals(self.counter, N + 1)

        for i in range(1, 1 + N):
            sem.release()
            self.assertEquals(self.counter, N + 1) 

def testSemaphore(self):
        N = 13
        sem = defer.DeferredSemaphore(N)

        controlDeferred = defer.Deferred()
        def helper(self, arg):
            self.arg = arg
            return controlDeferred

        results = []
        uniqueObject = object()
        resultDeferred = sem.run(helper, self=self, arg=uniqueObject)
        resultDeferred.addCallback(results.append)
        resultDeferred.addCallback(self._incr)
        self.assertEqual(results, [])
        self.assertEqual(self.arg, uniqueObject)
        controlDeferred.callback(None)
        self.assertIsNone(results.pop())
        self.assertEqual(self.counter, 1)

        self.counter = 0
        for i in range(1, 1 + N):
            sem.acquire().addCallback(self._incr)
            self.assertEqual(self.counter, i)


        success = []
        def fail(r):
            success.append(False)
        def succeed(r):
            success.append(True)
        d = sem.acquire().addCallbacks(fail, succeed)
        d.cancel()
        self.assertEqual(success, [True])

        sem.acquire().addCallback(self._incr)
        self.assertEqual(self.counter, N)

        sem.release()
        self.assertEqual(self.counter, N + 1)

        for i in range(1, 1 + N):
            sem.release()
            self.assertEqual(self.counter, N + 1) 

def testSemaphore(self):
        N = 13
        sem = defer.DeferredSemaphore(N)

        controlDeferred = defer.Deferred()
        def helper(self, arg):
            self.arg = arg
            return controlDeferred

        results = []
        uniqueObject = object()
        resultDeferred = sem.run(helper, self=self, arg=uniqueObject)
        resultDeferred.addCallback(results.append)
        resultDeferred.addCallback(self._incr)
        self.assertEqual(results, [])
        self.assertEqual(self.arg, uniqueObject)
        controlDeferred.callback(None)
        self.assertIsNone(results.pop())
        self.assertEqual(self.counter, 1)

        self.counter = 0
        for i in range(1, 1 + N):
            sem.acquire().addCallback(self._incr)
            self.assertEqual(self.counter, i)


        success = []
        def fail(r):
            success.append(False)
        def succeed(r):
            success.append(True)
        d = sem.acquire().addCallbacks(fail, succeed)
        d.cancel()
        self.assertEqual(success, [True])

        sem.acquire().addCallback(self._incr)
        self.assertEqual(self.counter, N)

        sem.release()
        self.assertEqual(self.counter, N + 1)

        for i in range(1, 1 + N):
            sem.release()
            self.assertEqual(self.counter, N + 1)