Python twisted.internet.threads.deferToThreadPool() Examples

def _recordsFromQueryString(
        self, queryString, recordTypes=None,
        limitResults=None, timeoutSeconds=None
    ):
        d = deferToThreadPool(
            reactor, self.threadpool,
            self._recordsFromQueryString_inThread,
            queryString,
            recordTypes,
            limitResults=limitResults,
            timeoutSeconds=timeoutSeconds
        )
        qsize = self.threadpool._queue.qsize()
        if qsize > 0:
            self.log.error("LDAP thread pool overflowing: {qsize}", qsize=qsize)
            self.poolStats["connection-thread-blocked"] += 1
        return d 

def getHostByName(self, name, timeout = (1, 3, 11, 45)):
        """
        See L{twisted.internet.interfaces.IResolverSimple.getHostByName}.

        Note that the elements of C{timeout} are summed and the result is used
        as a timeout for the lookup.  Any intermediate timeout or retry logic
        is left up to the platform via L{socket.gethostbyname}.
        """
        if timeout:
            timeoutDelay = sum(timeout)
        else:
            timeoutDelay = 60
        userDeferred = defer.Deferred()
        lookupDeferred = threads.deferToThreadPool(
            self.reactor, self.reactor.getThreadPool(),
            socket.gethostbyname, name)
        cancelCall = self.reactor.callLater(
            timeoutDelay, self._cleanup, name, lookupDeferred)
        self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
        lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred)
        return userDeferred 

def runWithConnection(self, func, *args, **kw):
        """
        Execute a function with a database connection and return the result.

        @param func: A callable object of one argument which will be executed
            in a thread with a connection from the pool.  It will be passed as
            its first argument a L{Connection} instance (whose interface is
            mostly identical to that of a connection object for your DB-API
            module of choice), and its results will be returned as a Deferred.
            If the method raises an exception the transaction will be rolled
            back.  Otherwise, the transaction will be committed.  B{Note} that
            this function is B{not} run in the main thread: it must be
            threadsafe.

        @param *args: positional arguments to be passed to func

        @param **kw: keyword arguments to be passed to func

        @return: a Deferred which will fire the return value of
            C{func(Transaction(...), *args, **kw)}, or a Failure.
        """
        from twisted.internet import reactor
        return threads.deferToThreadPool(reactor, self.threadpool,
                                         self._runWithConnection,
                                         func, *args, **kw) 

def _deferToThreadPool(self, f, *args, **kwargs):
        """Defer execution of ``f(*args, **kwargs)`` to the thread pool.
        This returns a deferred which will callback with the result of
        that expression, or errback with a failure wrapping the raised
        exception.
        """
        if self._pool.joined:
            return fail(
                ReactorNotRunning("This thimble's threadpool already stopped.")
            )
        if not self._pool.started:
            self._pool.start()
            self._reactor.addSystemEventTrigger(
                'during', 'shutdown', self._pool.stop)

        return deferToThreadPool(self._reactor, self._pool, f, *args, **kwargs) 

def runWithConnection(self, func, *args, **kw):
        """
        Execute a function with a database connection and return the result.

        @param func: A callable object of one argument which will be executed
            in a thread with a connection from the pool. It will be passed as
            its first argument a L{Connection} instance (whose interface is
            mostly identical to that of a connection object for your DB-API
            module of choice), and its results will be returned as a
            L{Deferred}. If the method raises an exception the transaction will
            be rolled back. Otherwise, the transaction will be committed.
            B{Note} that this function is B{not} run in the main thread: it
            must be threadsafe.

        @param *args: positional arguments to be passed to func

        @param **kw: keyword arguments to be passed to func

        @return: a L{Deferred} which will fire the return value of
            C{func(Transaction(...), *args, **kw)}, or a
            L{twisted.python.failure.Failure}.
        """
        return threads.deferToThreadPool(self._reactor, self.threadpool,
                                         self._runWithConnection,
                                         func, *args, **kw) 

def getHostByName(self, name, timeout = (1, 3, 11, 45)):
        """
        See L{twisted.internet.interfaces.IResolverSimple.getHostByName}.

        Note that the elements of C{timeout} are summed and the result is used
        as a timeout for the lookup.  Any intermediate timeout or retry logic
        is left up to the platform via L{socket.gethostbyname}.
        """
        if timeout:
            timeoutDelay = sum(timeout)
        else:
            timeoutDelay = 60
        userDeferred = defer.Deferred()
        lookupDeferred = threads.deferToThreadPool(
            self.reactor, self.reactor.getThreadPool(),
            socket.gethostbyname, name)
        cancelCall = self.reactor.callLater(
            timeoutDelay, self._cleanup, name, lookupDeferred)
        self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
        lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred)
        return userDeferred 

def _authenticateUsernamePassword(self, dn, password):
        """
        Open a secondary connection to the LDAP server and try binding to it
        with the given credentials

        @returns: True if the password is correct, False otherwise
        @rtype: deferred C{bool}

        @raises: L{LDAPConnectionError} if unable to connect.
        """
        d = deferToThreadPool(
            reactor, self.threadpool,
            self._authenticateUsernamePassword_inThread, dn, password
        )
        qsize = self.threadpool._queue.qsize()
        if qsize > 0:
            self.log.error("LDAP thread pool overflowing: {qsize}", qsize=qsize)
            self.poolStats["connection-thread-blocked"] += 1
        return d 

def getHostByName(self, name, timeout = (1, 3, 11, 45)):
        """
        See L{twisted.internet.interfaces.IResolverSimple.getHostByName}.

        Note that the elements of C{timeout} are summed and the result is used
        as a timeout for the lookup.  Any intermediate timeout or retry logic
        is left up to the platform via L{socket.gethostbyname}.
        """
        if timeout:
            timeoutDelay = sum(timeout)
        else:
            timeoutDelay = 60
        userDeferred = defer.Deferred()
        lookupDeferred = threads.deferToThreadPool(
            self.reactor, self.reactor.getThreadPool(),
            socket.gethostbyname, name)
        cancelCall = self.reactor.callLater(
            timeoutDelay, self._cleanup, name, lookupDeferred)
        self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
        lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred)
        return userDeferred 

def test_context_entry_failures_are_propagated_to_tasks(self):
        exception = factory.make_exception()

        pool = ThreadPool(
            contextFactory=partial(ContextBrokenOnEntry, exception),
            minthreads=1,
            maxthreads=1,
        )
        self.addCleanup(stop_pool_if_running, pool)
        pool.start()

        d = deferToThreadPool(reactor, pool, lambda: None)
        return assert_fails_with(d, type(exception)) 

def runInteraction(self, interaction, *args, **kw):
        """
        Interact with the database and return the result.

        The 'interaction' is a callable object which will be executed in a
        thread using a pooled connection. It will be passed an L{Transaction}
        object as an argument (whose interface is identical to that of the
        database cursor for your DB-API module of choice), and its results will
        be returned as a L{Deferred}. If running the method raises an
        exception, the transaction will be rolled back. If the method returns a
        value, the transaction will be committed.

        NOTE that the function you pass is *not* run in the main thread: you
        may have to worry about thread-safety in the function you pass to this
        if it tries to use non-local objects.

        @param interaction: a callable object whose first argument is an
            L{adbapi.Transaction}.

        @param *args: additional positional arguments to be passed to
            interaction

        @param **kw: keyword arguments to be passed to interaction

        @return: a Deferred which will fire the return value of
            C{interaction(Transaction(...), *args, **kw)}, or a
            L{twisted.python.failure.Failure}.
        """
        return threads.deferToThreadPool(self._reactor, self.threadpool,
                                         self._runInteraction,
                                         interaction, *args, **kw) 

def test_deferredFailure(self):
        """
        Check that L{threads.deferToThreadPool} return a failure object with an
        appropriate exception instance when the called function raises an
        exception.
        """
        class NewError(Exception):
            pass
        def raiseError():
            raise NewError()
        d = threads.deferToThreadPool(reactor, self.tp, raiseError)
        return self.assertFailure(d, NewError) 

def test_deferredResult(self):
        """
        L{threads.deferToThreadPool} executes the function passed, and
        correctly handles the positional and keyword arguments given.
        """
        d = threads.deferToThreadPool(reactor, self.tp,
                                      lambda x, y=5: x + y, 3, y=4)
        d.addCallback(self.assertEqual, 7)
        return d 

def test_context_exit_failures_are_logged(self):
        exception = factory.make_exception()

        pool = ThreadPool(
            contextFactory=partial(ContextBrokenOnExit, exception),
            minthreads=1,
            maxthreads=1,
        )
        self.addCleanup(stop_pool_if_running, pool)
        pool.start()

        result = yield deferToThreadPool(reactor, pool, lambda: sentinel.foo)
        self.assertThat(result, Is(sentinel.foo))

        with TwistedLoggerFixture() as logger:
            pool.stop()

        self.assertThat(
            logger.output,
            DocTestMatches(
                """\
            Failure exiting worker context.
            Traceback (most recent call last):
            ...
            maastesting.factory.TestException#...
            """
            ),
        ) 

def test_when_deferring_acquires_and_releases_lock(self):
        pool = self.make_pool()
        # Within the thread the lock had been acquired.
        tokens_in_thread = yield deferToThreadPool(
            reactor, pool, lambda: pool.lock.tokens
        )
        self.assertThat(tokens_in_thread, Equals(0))
        # The lock has not yet been released.
        self.assertThat(pool.lock.tokens, Equals(0))
        # Wait and it shall be released.
        yield pool.lock.run(noop)
        self.assertThat(pool.lock.tokens, Equals(1)) 

def test_when_deferring_acquires_and_releases_lock_on_error(self):
        pool = self.make_pool()
        # Within the thread the lock had been acquired.
        with ExpectedException(ZeroDivisionError):
            yield deferToThreadPool(reactor, pool, lambda: 0 / 0)
        # The lock has not yet been released.
        self.assertThat(pool.lock.tokens, Equals(0))
        # Wait and it shall be released.
        yield pool.lock.run(noop)
        self.assertThat(pool.lock.tokens, Equals(1)) 

def runInteraction(self, interaction, *args, **kw):
        """
        Interact with the database and return the result.

        The 'interaction' is a callable object which will be executed
        in a thread using a pooled connection. It will be passed an
        L{Transaction} object as an argument (whose interface is
        identical to that of the database cursor for your DB-API
        module of choice), and its results will be returned as a
        Deferred. If running the method raises an exception, the
        transaction will be rolled back. If the method returns a
        value, the transaction will be committed.

        NOTE that the function you pass is *not* run in the main
        thread: you may have to worry about thread-safety in the
        function you pass to this if it tries to use non-local
        objects.

        @param interaction: a callable object whose first argument
            is an L{adbapi.Transaction}.

        @param *args: additional positional arguments to be passed
            to interaction

        @param **kw: keyword arguments to be passed to interaction

        @return: a Deferred which will fire the return value of
            'interaction(Transaction(...), *args, **kw)', or a Failure.
        """
        from twisted.internet import reactor
        return threads.deferToThreadPool(reactor, self.threadpool,
                                         self._runInteraction,
                                         interaction, *args, **kw) 

def augment_twisted_deferToThreadPool():
    """Wrap every function deferred to a thread in `synchronous`."""
    from twisted.internet import threads
    from twisted.internet.threads import deferToThreadPool
    from provisioningserver.utils.twisted import ISynchronous, synchronous

    def new_deferToThreadPool(reactor, threadpool, f, *args, **kwargs):
        """Variant of Twisted's that wraps all functions in `synchronous`."""
        func = f if ISynchronous.providedBy(f) else synchronous(f)
        return deferToThreadPool(reactor, threadpool, func, *args, **kwargs)

    if threads.deferToThreadPool.__module__ != __name__:
        threads.deferToThreadPool = new_deferToThreadPool 

def stopService(self):
        """
        Stop the writer thread, wait for it to finish.
        """
        Service.stopService(self)
        removeDestination(self)
        self._reactor.callFromThread(self._reactor.stop)
        return deferToThreadPool(
            self._mainReactor, self._mainReactor.getThreadPool(), self._thread.join
        ) 

def test_deferredResult(self):
        """
        L{threads.deferToThreadPool} executes the function passed, and
        correctly handles the positional and keyword arguments given.
        """
        d = threads.deferToThreadPool(reactor, self.tp,
                                      lambda x, y=5: x + y, 3, y=4)
        d.addCallback(self.assertEqual, 7)
        return d 

def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
                        addressTypes=None, transportSemantics='TCP'):
        """
        See L{IHostnameResolver.resolveHostName}

        @param resolutionReceiver: see interface

        @param hostName: see interface

        @param portNumber: see interface

        @param addressTypes: see interface

        @param transportSemantics: see interface

        @return: see interface
        """
        pool = self._getThreadPool()
        addressFamily = _typesToAF[_any if addressTypes is None
                                   else frozenset(addressTypes)]
        socketType = _transportToSocket[transportSemantics]
        def get():
            try:
                return self._getaddrinfo(hostName, portNumber, addressFamily,
                                         socketType)
            except gaierror:
                return []
        d = deferToThreadPool(self._reactor, pool, get)
        resolution = HostResolution(hostName)
        resolutionReceiver.resolutionBegan(resolution)
        @d.addCallback
        def deliverResults(result):
            for family, socktype, proto, cannoname, sockaddr in result:
                addrType = _afToType[family]
                resolutionReceiver.addressResolved(
                    addrType(_socktypeToType.get(socktype, 'TCP'), *sockaddr)
                )
            resolutionReceiver.resolutionComplete()
        return resolution 

def _recordWithDN(self, dn):
        d = deferToThreadPool(
            reactor, self.threadpool,
            self._recordWithDN_inThread, dn
        )
        qsize = self.threadpool._queue.qsize()
        if qsize > 0:
            self.log.error("LDAP thread pool overflowing: {qsize}", qsize=qsize)
            self.poolStats["connection-thread-blocked"] += 1
        return d 

def _threaded_method(method_name, sync_name, reactor_name, threadpool_name):
    """
    Create a method that calls another method in a threadpool.

    :param str method_name: The name of the method to look up on the "sync"
        object.
    :param str sync_name: The name of the attribute of ``self`` on which to
        look up the other method to run.  This is the "sync" object.
    :param str reactor_name: The name of the attribute of ``self`` referencing
        the reactor to use to get results back to the calling thread.
    :param str threadpool_name: The name of the attribute of ``self``
        referencing a ``twisted.python.threadpool.ThreadPool`` instance to use
        to run the method in a different thread.

    :return: The new thread-using method.  It has the same signature as the
             original method except it returns a ``Deferred`` that fires with
             the original method's result.
    """
    def _run_in_thread(self, *args, **kwargs):
        reactor = getattr(self, reactor_name)
        sync = getattr(self, sync_name)
        threadpool = getattr(self, threadpool_name)
        original = getattr(sync, method_name)
        return deferToThreadPool(
            reactor, threadpool, preserve_context(original), *args, **kwargs
        )
    return _run_in_thread 

def test_deferredResult(self):
        """
        L{threads.deferToThreadPool} executes the function passed, and
        correctly handles the positional and keyword arguments given.
        """
        d = threads.deferToThreadPool(reactor, self.tp,
                                      lambda x, y=5: x + y, 3, y=4)
        d.addCallback(self.assertEqual, 7)
        return d 

def runInteraction(self, interaction, *args, **kw):
        """
        Interact with the database and return the result.

        The 'interaction' is a callable object which will be executed in a
        thread using a pooled connection. It will be passed an L{Transaction}
        object as an argument (whose interface is identical to that of the
        database cursor for your DB-API module of choice), and its results will
        be returned as a L{Deferred}. If running the method raises an
        exception, the transaction will be rolled back. If the method returns a
        value, the transaction will be committed.

        NOTE that the function you pass is *not* run in the main thread: you
        may have to worry about thread-safety in the function you pass to this
        if it tries to use non-local objects.

        @param interaction: a callable object whose first argument is an
            L{adbapi.Transaction}.

        @param *args: additional positional arguments to be passed to
            interaction

        @param **kw: keyword arguments to be passed to interaction

        @return: a Deferred which will fire the return value of
            C{interaction(Transaction(...), *args, **kw)}, or a
            L{twisted.python.failure.Failure}.
        """
        from twisted.internet import reactor
        return threads.deferToThreadPool(reactor, self.threadpool,
                                         self._runInteraction,
                                         interaction, *args, **kw) 

def runWithConnection(self, func, *args, **kw):
        """
        Execute a function with a database connection and return the result.

        @param func: A callable object of one argument which will be executed
            in a thread with a connection from the pool. It will be passed as
            its first argument a L{Connection} instance (whose interface is
            mostly identical to that of a connection object for your DB-API
            module of choice), and its results will be returned as a
            L{Deferred}. If the method raises an exception the transaction will
            be rolled back. Otherwise, the transaction will be committed.
            B{Note} that this function is B{not} run in the main thread: it
            must be threadsafe.

        @param *args: positional arguments to be passed to func

        @param **kw: keyword arguments to be passed to func

        @return: a L{Deferred} which will fire the return value of
            C{func(Transaction(...), *args, **kw)}, or a
            L{twisted.python.failure.Failure}.
        """
        from twisted.internet import reactor
        return threads.deferToThreadPool(reactor, self.threadpool,
                                         self._runWithConnection,
                                         func, *args, **kw) 

def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
                        addressTypes=None, transportSemantics='TCP'):
        """
        See L{IHostnameResolver.resolveHostName}

        @param resolutionReceiver: see interface

        @param hostName: see interface

        @param portNumber: see interface

        @param addressTypes: see interface

        @param transportSemantics: see interface

        @return: see interface
        """
        pool = self._getThreadPool()
        addressFamily = _typesToAF[_any if addressTypes is None
                                   else frozenset(addressTypes)]
        socketType = _transportToSocket[transportSemantics]
        def get():
            try:
                return self._getaddrinfo(hostName, portNumber, addressFamily,
                                         socketType)
            except gaierror:
                return []
        d = deferToThreadPool(self._reactor, pool, get)
        resolution = HostResolution(hostName)
        resolutionReceiver.resolutionBegan(resolution)
        @d.addCallback
        def deliverResults(result):
            for family, socktype, proto, cannoname, sockaddr in result:
                addrType = _afToType[family]
                resolutionReceiver.addressResolved(
                    addrType(_socktypeToType.get(socktype, 'TCP'), *sockaddr)
                )
            resolutionReceiver.resolutionComplete()
        return resolution