Python twisted.python.failure.Failure() Examples

def test_producer_stop_during_request(self):
        """
        Test stopping producer while it's waiting for reply from client
        """
        clock = MemoryReactorClock()
        client = Mock(reactor=clock)
        f = Failure(BrokerNotAvailableError())
        ret = [fail(f), Deferred()]
        client.send_produce_request.side_effect = ret
        client.topic_partitions = {self.topic: [0, 1, 2, 3]}
        client.metadata_error_for_topic.return_value = False
        msgs = [self.msg("one"), self.msg("two")]
        batch_n = 2

        producer = Producer(client, batch_every_n=batch_n, batch_send=True)
        d = producer.send_messages(self.topic, msgs=msgs)
        # At first, there's no result. Have to retry due to first failure
        self.assertNoResult(d)
        clock.advance(producer._retry_interval)

        producer.stop()
        self.failureResultOf(d, tid_CancelledError) 

def testIllegalInboxDelete(self):
        """
        Test what happens if we try to delete the user Inbox.
        We expect that operation to fail.
        """
        self.stashed = None

        def login():
            return self.client.login(TEST_USER, TEST_PASSWD)

        def delete():
            return self.client.delete('inbox')

        def stash(result):
            self.stashed = result

        d1 = self.connected.addCallback(strip(login))
        d1.addCallbacks(strip(delete), self._ebGeneral)
        d1.addBoth(stash)
        d1.addCallbacks(self._cbStopClient, self._ebGeneral)
        d2 = self.loopback()
        d = defer.gatherResults([d1, d2])
        d.addCallback(lambda _: self.failUnless(isinstance(self.stashed,
                                                           failure.Failure)))
        return d 

def test_returnsBytes(self):
        """
        The return value of L{formatFailure} is a C{str} instance (not a
        C{unicode} instance) with numeric character references for any non-ASCII
        characters meant to appear in the output.
        """
        try:
            raise Exception("Fake bug")
        except:
            result = formatFailure(Failure())

        self.assertIsInstance(result, bytes)
        if _PY3:
            self.assertTrue(all(ch < 128 for ch in result))
        else:
            self.assertTrue(all(ord(ch) < 128 for ch in result))
        # Indentation happens to rely on NO-BREAK SPACE
        self.assertIn(b"&#160;", result) 

def test_renderNoFailure(self):
        """
        If the L{Deferred} fails, L{DeferredResource} reports the failure via
        C{processingFailed}, and does not cause an unhandled error to be
        logged.
        """
        request = DummyRequest([])
        d = request.notifyFinish()
        failure = Failure(RuntimeError())
        deferredResource = DeferredResource(defer.fail(failure))
        deferredResource.render(request)
        self.assertEqual(self.failureResultOf(d), failure)
        del deferredResource
        gc.collect()
        errors = self.flushLoggedErrors(RuntimeError)
        self.assertEqual(errors, []) 

def assertFlattensImmediately(self, root, target):
        """
        Assert that a root element, when flattened, is equal to a string, and
        performs no asynchronus Deferred anything.

        This version is more convenient in tests which wish to make multiple
        assertions about flattening, since it can be called multiple times
        without having to add multiple callbacks.

        @return: the result of rendering L{root}, which should be equivalent to
            L{target}.
        @rtype: L{bytes}
        """
        results = []
        it = self.assertFlattensTo(root, target)
        it.addBoth(results.append)
        # Do our best to clean it up if something goes wrong.
        self.addCleanup(it.cancel)
        if not results:
            self.fail("Rendering did not complete immediately.")
        result = results[0]
        if isinstance(result, Failure):
            result.raiseException()
        return results[0] 

def _disconnectParser(self, reason):
        """
        If there is still a parser, call its C{connectionLost} method with the
        given reason.  If there is not, do nothing.

        @type reason: L{Failure}
        """
        if self._parser is not None:
            parser = self._parser
            self._parser = None
            self._currentRequest = None
            self._finishedRequest = None
            self._responseDeferred = None

            # The parser is no longer allowed to do anything to the real
            # transport.  Stop proxying from the parser's transport to the real
            # transport before telling the parser it's done so that it can't do
            # anything.
            self._transportProxy._stopProxying()
            self._transportProxy = None
            parser.connectionLost(reason) 

def send(self, cmd, args, expect_ack=None, ack_callback=None, response_callback=None, expect_status=None, status_callback=None, update_callback=None, update_handler=None):
        if self.ready:
            self.cmd = cmd
            if 'request' in cmd:
                self.response_rows = []
            ret = self.api.gateway_send('%s %s %s' % (cmd, self.id, args))
            self.ack_pending = expect_ack
            self.ack_callback = ack_callback
            self.response_pending = bool(response_callback)
            self.response_callback = response_callback
            self.status_pending = expect_status
            self.status_callback = status_callback
            self.update_callback = update_callback
            self.update_handler = update_handler
        else:
            if self.on_connect_action:
                self.api.error_handler(self.id, 'Failure: on_connect_action already exists: %s' % repr(self.on_connect_action))
                ret = False
            else:
                self.api.output('%s storing on_connect_action (%s)...' % (self, cmd))
                self.on_connect_action = (cmd, args, expect_ack, ack_callback, response_callback, expect_status, status_callback, update_callback, update_handler)
                ret = True
        return ret 

def writeTo(self, transport):
        """
        Format this L{Request} as an HTTP/1.1 request and write it to the given
        transport.  If bodyProducer is not None, it will be associated with an
        L{IConsumer}.

        @param transport: The transport to which to write.
        @type transport: L{twisted.internet.interfaces.ITransport} provider

        @return: A L{Deferred} which fires with L{None} when the request has
            been completely written to the transport or with a L{Failure} if
            there is any problem generating the request bytes.
        """
        if self.bodyProducer is None:
            # If the method semantics anticipate a body, include a
            # Content-Length even if it is 0.
            # https://tools.ietf.org/html/rfc7230#section-3.3.2
            if self.method in (b"PUT", b"POST"):
                self._writeToEmptyBodyContentLength(transport)
            else:
                self._writeHeaders(transport, None)
        elif self.bodyProducer.length is UNKNOWN_LENGTH:
            return self._writeToBodyProducerChunked(transport)
        else:
            return self._writeToBodyProducerContentLength(transport) 

def _complete_batch_send(self, resp):
        """Complete the processing of our batch send operation

        Clear the deferred tracking our current batch processing
        and reset our retry count and retry interval
        Return none to eat any errors coming from up the deferred chain
        """
        self._batch_send_d = None
        self._req_attempts = 0
        self._retry_interval = self._init_retry_interval
        if isinstance(resp, Failure) and not resp.check(tid_CancelledError,
                                                        CancelledError):
            log.error(
                "Failure detected in _complete_batch_send: %r", resp,
                exc_info=(resp.type, resp.value, resp.getTracebackObject()),
            )
        return 

def _handleRedirect(self, response, method, uri, headers, redirectCount):
        """
        Handle a redirect response, checking the number of redirects already
        followed, and extracting the location header fields.
        """
        if redirectCount >= self._redirectLimit:
            err = error.InfiniteRedirection(
                response.code,
                b'Infinite redirection detected',
                location=uri)
            raise ResponseFailed([Failure(err)], response)
        locationHeaders = response.headers.getRawHeaders(b'location', [])
        if not locationHeaders:
            err = error.RedirectWithNoLocation(
                response.code, b'No location header field', uri)
            raise ResponseFailed([Failure(err)], response)
        location = self._resolveLocation(uri, locationHeaders[0])
        deferred = self._agent.request(method, location, headers)
        def _chainResponse(newResponse):
            newResponse.setPreviousResponse(response)
            return newResponse
        deferred.addCallback(_chainResponse)
        return deferred.addCallback(
            self._handleResponse, method, uri, headers, redirectCount + 1) 

def test_consumer_error_during_offset(self):
        topic = 'error_during_offset'
        part = 991
        reqs_ds = [Deferred(), Deferred()]
        clock = MemoryReactorClock()
        mockclient = Mock(reactor=clock)
        mockclient.send_offset_request.side_effect = reqs_ds
        consumer = Consumer(mockclient, topic, part, Mock())
        d = consumer.start(OFFSET_LATEST)
        # Make sure request for offset was made
        request = OffsetRequest(topic, part, OFFSET_LATEST, 1)
        mockclient.send_offset_request.assert_called_once_with([request])
        # Errback the first request
        f = Failure(KafkaUnavailableError())  # Perhaps kafka wasn't up yet...
        with patch.object(kconsumer, 'log'):
            reqs_ds[0].errback(f)
        # Advance the clock to trigger the 2nd request
        clock.advance(consumer.retry_delay + 1)  # fire the callLater
        self.assertEqual(2, mockclient.send_offset_request.call_count)

        # Stop the consumer to cleanup any outstanding operations
        self.assertIsNone(consumer.stop())
        self.assertIsNone(self.successResultOf(d)) 

def test_consumer_offset_out_of_range_error_without_reset(self):
        topic = 'offset_out_of_range_error'
        part = 911
        offset = 10000
        fetch_ds = [Deferred()]
        clock = MemoryReactorClock()
        mockclient = Mock(reactor=clock)
        mockclient.send_fetch_request.side_effect = fetch_ds

        consumer = Consumer(mockclient, topic, part, Mock())
        d = consumer.start(offset)

        f = Failure(OffsetOutOfRangeError())
        fetch_ds[0].errback(f)

        self.assertEqual(self.failureResultOf(d), f)

        consumer.stop() 

def handleResponse(self, response):
        if self.quietLoss:
            return
        if self.failed:
            self.factory.noPage(
                Failure(
                    error.Error(
                        self.status, self.message, response)))
        if self.factory.method == b'HEAD':
            # Callback with empty string, since there is never a response
            # body for HEAD requests.
            self.factory.page(b'')
        elif self.length != None and self.length != 0:
            self.factory.noPage(Failure(
                PartialDownloadError(self.status, self.message, response)))
        else:
            self.factory.page(response)
        # server might be stupid and not close connection. admittedly
        # the fact we do only one request per connection is also
        # stupid...
        self.transport.loseConnection() 

def process(self):
        """
        Process a request.
        """

        # get site from channel
        self.site = self.channel.site

        # set various default headers
        self.setHeader(b'server', version)
        self.setHeader(b'date', http.datetimeToString())

        # Resource Identification
        self.prepath = []
        self.postpath = list(map(unquote, self.path[1:].split(b'/')))

        try:
            resrc = self.site.getResourceFor(self)
            if resource._IEncodingResource.providedBy(resrc):
                encoder = resrc.getEncoder(self)
                if encoder is not None:
                    self._encoder = encoder
            self.render(resrc)
        except:
            self.processingFailed(failure.Failure()) 

def test_consumer_error(self):
        """
            get an unexpected stop error from a consumer
        """
        client = self.mock_client([])
        processor = Mock()
        group = ConsumerGroup(client, "group_id", "topic1", processor)
        start_d = group.start()
        self.assertNoResult(start_d)
        with patch('afkak._group.Consumer') as mock_consumer:
            mock_consumer.return_value.start.return_value = d = defer.Deferred()
            group.on_join_complete({"topic1": [1]})
            self.assertEqual(mock_consumer.return_value.start.called, True)
            d.errback(Failure(AssertionError()))
            self.failureResultOf(start_d, AssertionError)
            d.addErrback(lambda result: None) 

def test_producer_send_messages_no_retry_fail(self):
        client = Mock(reactor=MemoryReactorClock())
        f = Failure(BrokerNotAvailableError())
        client.send_produce_request.side_effect = [fail(f)]
        client.topic_partitions = {self.topic: [0, 1, 2, 3]}
        client.metadata_error_for_topic.return_value = False
        msgs = [self.msg("one"), self.msg("two")]

        producer = Producer(client, max_req_attempts=1)
        d = producer.send_messages(self.topic, msgs=msgs)
        # Check the expected request was sent
        msgSet = create_message_set(
            make_send_requests(msgs), producer.codec)
        req = ProduceRequest(self.topic, 0, msgSet)
        client.send_produce_request.assert_called_once_with(
            [req], acks=producer.req_acks, timeout=producer.ack_timeout,
            fail_on_error=False)
        self.failureResultOf(d, BrokerNotAvailableError)

        producer.stop() 

def test_producer_send_messages_unexpected_err(self):
        client = Mock(reactor=MemoryReactorClock())
        f = Failure(TypeError())
        client.send_produce_request.side_effect = [fail(f)]
        client.topic_partitions = {self.topic: [0, 1, 2, 3]}
        client.metadata_error_for_topic.return_value = False
        msgs = [self.msg("one"), self.msg("two")]

        producer = Producer(client)
        # FIXME: Don't use patch to test logging
        with patch.object(aProducer, 'log') as klog:
            d = producer.send_messages(self.topic, msgs=msgs)
            klog.error.assert_called_once_with(
                'Unexpected failure: %r in _handle_send_response', f)
        self.failureResultOf(d, TypeError)

        producer.stop() 

def test_producer_stop_waiting_to_retry(self):
        """
        Test stopping producer while it's waiting to retry a request
        """
        clock = MemoryReactorClock()
        client = Mock(reactor=clock)
        f = Failure(BrokerNotAvailableError())
        ret = [fail(f)]
        client.send_produce_request.side_effect = ret
        client.topic_partitions = {self.topic: [0, 1, 2, 3]}
        client.metadata_error_for_topic.return_value = False
        msgs = [self.msg("one"), self.msg("two")]
        batch_n = 2

        producer = Producer(client, batch_every_n=batch_n, batch_send=True)
        d = producer.send_messages(self.topic, msgs=msgs)
        # At first, there's no result. Have to retry due to first failure
        self.assertNoResult(d)
        # Advance the clock, some, but not enough to retry
        clock.advance(producer._retry_interval / 2)
        # Stop the producer before the retry
        producer.stop()
        self.failureResultOf(d, tid_CancelledError) 

def set_order_route(self, route, callback):
        #print('set_order_route(%s, %s) type=%s %s' % (repr(route), repr(callback), type(route), (type(route) in [str, unicode])))
        if type(route) in [str, unicode]:
            if route.startswith('{'):
                route = json.loads(route)
	    elif route.startswith('"'):
                route = {json.loads(route): None}
            else:
                route = {route: None}
        if (type(route)==dict) and (len(route.keys()) == 1) and (type(route.keys()[0]) in [str, unicode]):
            self.order_route = route
            if callback:
                self.get_order_route(callback)
        else:
            if callback:
                callback.errback(Failure(Exception('cannot set order route %s' % route)))
            else:
                self.error_handler(None, 'Cannot set order route %s' % repr(route)) 

def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Issue a request to the server indicated by the given C{uri}.

        An existing connection from the connection pool may be used or a new
        one may be created.

        I{HTTP} and I{HTTPS} schemes are supported in C{uri}.

        @see: L{twisted.web.iweb.IAgent.request}
        """
        parsedURI = URI.fromBytes(uri)
        try:
            endpoint = self._getEndpoint(parsedURI)
        except SchemeNotSupported:
            return defer.fail(Failure())
        key = (parsedURI.scheme, parsedURI.host, parsedURI.port)
        return self._requestWithEndpoint(key, endpoint, method, parsedURI,
                                         headers, bodyProducer,
                                         parsedURI.originForm) 

def test_failedReadWhileProducing(self):
        """
        If a read from the input file fails while producing bytes to the
        consumer, the L{Deferred} returned by
        L{FileBodyProducer.startProducing} fires with a L{Failure} wrapping
        that exception.
        """
        class BrokenFile(object):
            def read(self, count):
                raise IOError("Simulated bad thing")
        producer = FileBodyProducer(BrokenFile(), self.cooperator)
        complete = producer.startProducing(FileConsumer(BytesIO()))
        self._scheduled.pop(0)()
        self.failureResultOf(complete).trap(IOError) 

def _handle_offset_error(self, failure):
        """
        Retry the offset fetch request if appropriate.

        Once the :attr:`.retry_delay` reaches our :attr:`.retry_max_delay`, we
        log a warning.  This should perhaps be extended to abort sooner on
        certain errors.
        """
        # outstanding request got errback'd, clear it
        self._request_d = None

        if self._stopping and failure.check(CancelledError):
            # Not really an error
            return
        # Do we need to abort?
        if (self.request_retry_max_attempts != 0 and
                self._fetch_attempt_count >= self.request_retry_max_attempts):
            log.debug(
                "%r: Exhausted attempts: %d fetching offset from kafka",
                self, self.request_retry_max_attempts,
                exc_info=(failure.type, failure.value, failure.getTracebackObject()),
            )
            self._start_d.errback(failure)
            return
        # Decide how to log this failure... If we have retried so many times
        # we're at the retry_max_delay, then we log at warning every other time
        # debug otherwise
        if (self.retry_delay < self.retry_max_delay or
                0 == (self._fetch_attempt_count % 2)):
            log.debug("%r: Failure fetching offset from kafka: %r", self,
                      failure)
        else:
            # We've retried until we hit the max delay, log at warn
            log.warning("%r: Still failing fetching offset from kafka: %r",
                        self, failure)
        self._retry_fetch() 

def test_connectionLost_cleanly(self):
        """
        The factory is notified of connection loss.
        """
        kp = KafkaProtocol()
        kp.factory = factory_spy = mock.Mock(wraps=TheFactory())
        reason = Failure(ConnectionDone())

        kp.connectionLost(reason)

        factory_spy._connectionLost.assert_called_once_with(reason)
        self.assertIsNone(kp.factory) 

def pagePart(self, data):
        if not self.file:
            return
        try:
            self.file.write(data)
        except IOError:
            #raise
            self.file = None
            self.deferred.errback(Failure()) 

def run(self):
        """
        Call the WSGI application object, iterate it, and handle its output.

        This must be called in a non-I/O thread (ie, a WSGI application
        thread).
        """
        try:
            appIterator = self.application(self.environ, self.startResponse)
            for elem in appIterator:
                if elem:
                    self.write(elem)
                if self._requestFinished:
                    break
            close = getattr(appIterator, 'close', None)
            if close is not None:
                close()
        except:
            def wsgiError(started, type, value, traceback):
                err(Failure(value, type, traceback), "WSGI application error")
                if started:
                    self.request.loseConnection()
                else:
                    self.request.setResponseCode(INTERNAL_SERVER_ERROR)
                    self.request.finish()
            self.reactor.callFromThread(wsgiError, self.started, *exc_info())
        else:
            def wsgiFinish(started):
                if not self._requestFinished:
                    if not started:
                        self._sendResponseHeaders()
                    self.request.finish()
            self.reactor.callFromThread(wsgiFinish, self.started)
        self.started = True 

def handleStatus_301(self):
        l = self.headers.get(b'location')
        if not l:
            self.handleStatusDefault()
            return
        url = l[0]
        if self.followRedirect:
            self.factory._redirectCount += 1
            if self.factory._redirectCount >= self.factory.redirectLimit:
                err = error.InfiniteRedirection(
                    self.status,
                    b'Infinite redirection detected',
                    location=url)
                self.factory.noPage(Failure(err))
                self.quietLoss = True
                self.transport.loseConnection()
                return

            self._completelyDone = False
            self.factory.setURL(url)

            if self.factory.scheme == b'https':
                from twisted.internet import ssl
                contextFactory = ssl.ClientContextFactory()
                reactor.connectSSL(nativeString(self.factory.host),
                                   self.factory.port,
                                   self.factory, contextFactory)
            else:
                reactor.connectTCP(nativeString(self.factory.host),
                                   self.factory.port,
                                   self.factory)
        else:
            self.handleStatusDefault()
            self.factory.noPage(
                Failure(
                    error.PageRedirect(
                        self.status, self.message, location = url)))
        self.quietLoss = True
        self.transport.loseConnection() 

def test_closeCachedConnections(self):
        """
        L{HTTPConnectionPool.closeCachedConnections} closes all cached
        connections and removes them from the cache. It returns a Deferred
        that fires when they have all lost their connections.
        """
        persistent = []
        def addProtocol(scheme, host, port):
            p = HTTP11ClientProtocol()
            p.makeConnection(StringTransport())
            self.pool._putConnection((scheme, host, port), p)
            persistent.append(p)
        addProtocol("http", b"example.com", 80)
        addProtocol("http", b"www2.example.com", 80)
        doneDeferred = self.pool.closeCachedConnections()

        # Connections have begun disconnecting:
        for p in persistent:
            self.assertEqual(p.transport.disconnecting, True)
        self.assertEqual(self.pool._connections, {})
        # All timeouts were cancelled and removed:
        for dc in self.fakeReactor.getDelayedCalls():
            self.assertEqual(dc.cancelled, True)
        self.assertEqual(self.pool._timeouts, {})

        # Returned Deferred fires when all connections have been closed:
        result = []
        doneDeferred.addCallback(result.append)
        self.assertEqual(result, [])
        persistent[0].connectionLost(Failure(ConnectionDone()))
        self.assertEqual(result, [])
        persistent[1].connectionLost(Failure(ConnectionDone()))
        self.assertEqual(result, [None]) 

def test_connectionFailed(self):
        """
        The L{Deferred} returned by L{Agent.request} fires with a L{Failure} if
        the TCP connection attempt fails.
        """
        result = self.agent.request(b'GET', b'http://foo/')
        # Cause the connection to be refused
        host, port, factory = self.reactor.tcpClients.pop()[:3]
        factory.clientConnectionFailed(None, Failure(ConnectionRefusedError()))
        self.reactor.advance(10)
        # ^ https://twistedmatrix.com/trac/ticket/8202
        self.failureResultOf(result, ConnectionRefusedError) 

def test_retryIfFailedDueToNonCancelException(self):
        """
        If a request failed with L{ResponseNeverReceived} due to some
        arbitrary exception, C{_shouldRetry} returns C{True} to indicate the
        request should be retried.
        """
        pool = client.HTTPConnectionPool(None)
        connection = client._RetryingHTTP11ClientProtocol(None, pool)
        self.assertTrue(connection._shouldRetry(
            b"GET", ResponseNeverReceived([Failure(Exception())]), None)) 

def _connectionClosedTest(self, application, responseContent):
        channel = DummyChannel()

        def applicationFactory():
            return application

        d, requestFactory = self.requestFactoryFactory()

        # Capture the request so we can disconnect it later on.
        requests = []
        def requestFactoryWrapper(*a, **kw):
            requests.append(requestFactory(*a, **kw))
            return requests[-1]

        def ebRendered(ignored):
            errors = self.flushLoggedErrors(RuntimeError)
            self.assertEqual(len(errors), 1)

            response = channel.transport.written.getvalue()
            self.assertTrue(response.startswith(b'HTTP/1.1 200 OK'))
            # Chunked transfer-encoding makes this a little messy.
            self.assertIn(responseContent, response)
        d.addErrback(ebRendered)

        self.lowLevelRender(
            requestFactoryWrapper, applicationFactory,
            lambda: channel, 'GET', '1.1', [], [''], None, [])

        # By now the connection should be closed.
        self.assertTrue(channel.transport.disconnected)
        # Give it a little push to go the rest of the way.
        requests[0].connectionLost(Failure(ConnectionLost("All gone")))

        return d