Java Code Examples for com.twitter.util.Await#result()

@Test(timeout = 60000)
public void testReaderLockSharedDlmDoesNotConflict() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogManager dlm0 = createNewDLM(conf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.write(DLMTestUtil.getLogRecordInstance(2L));
    writer.closeAndComplete();

    DistributedLogManager dlm1 = createNewDLM(conf, name);
    Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    Future<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);

    // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the
    // system currently works.
    Await.result(futureReader1);
    Await.result(futureReader2);

    dlm0.close();
    dlm1.close();
}
 

@Test
public void testFailedDueToClosed() throws Exception {
    TestFuturePool<Void> pool = new TestFuturePool<Void>();
    pool.wrapper.close();
    Future<Void> future = pool.wrapper.apply(new Function0<Void>() {
        public Void apply() {
            throw new RuntimeException("failed");
        }
    });
    try {
        Await.result(future);
        fail("should have thrown");
    } catch (RejectedExecutionException ex) {
    }
    pool.shutdown();
}
 

@Test(timeout = 60000)
public void testFailRequestsDuringClosing() throws Exception {
    String streamName = testName.getMethodName();
    StreamImpl s = createUnstartedStream(service, streamName);

    Future<Void> closeFuture = s.requestClose("close");
    assertTrue("Stream " + streamName + " should be set to " + StreamStatus.CLOSING,
            StreamStatus.CLOSING == s.getStatus() ||
            StreamStatus.CLOSED == s.getStatus());
    WriteOp op1 = createWriteOp(service, streamName, 0L);
    s.submit(op1);
    WriteResponse response1 = Await.result(op1.result());
    assertEquals("Op should fail with " + StatusCode.STREAM_UNAVAILABLE + " if it is closing",
            StatusCode.STREAM_UNAVAILABLE, response1.getHeader().getCode());

    Await.result(closeFuture);
    assertEquals("Stream " + streamName + " should be set to " + StreamStatus.CLOSED,
            StreamStatus.CLOSED, s.getStatus());
    WriteOp op2 = createWriteOp(service, streamName, 1L);
    s.submit(op2);
    WriteResponse response2 = Await.result(op2.result());
    assertEquals("Op should fail with " + StatusCode.STREAM_UNAVAILABLE + " if it is closed",
            StatusCode.STREAM_UNAVAILABLE, response2.getHeader().getCode());
}
 

/**
 * Non durable write should fail if writer is closed.
 *
 * @throws Exception
 */
@Test(timeout = 60000)
public void testNondurableWriteAfterWriterIsClosed() throws Exception {
    DistributedLogConfiguration confLocal = newLocalConf();
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setOutputBufferSize(Integer.MAX_VALUE);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setDurableWriteEnabled(false);
    ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true);
    BKLogSegmentWriter writer =
            createLogSegmentWriter(confLocal, 0L, -1L, lock);

    // close the writer
    closeWriterAndLock(writer, lock);
    FutureUtils.result(writer.asyncClose());

    try {
        Await.result(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(1)));
        fail("Should fail the write if the writer is closed");
    } catch (WriteException we) {
        // expected
    }
}
 

@Test(timeout = 20000)
public void testFailRequestAfterRetriedAllStreams() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any()))
            .thenReturn(new Promise<DLSN>());
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder()
            .streams(Lists.newArrayList("stream1", "stream2"))
            .client(client)
            .compressionCodec(CompressionCodec.Type.LZ4)
            .firstSpeculativeTimeoutMs(10)
            .maxSpeculativeTimeoutMs(20)
            .speculativeBackoffMultiplier(2)
            .requestTimeoutMs(5000000)
            .flushIntervalMs(10)
            .bufferSize(Integer.MAX_VALUE)
            .build();

    byte[] data = "test-test".getBytes(UTF_8);
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    try {
        Await.result(writeFuture);
        fail("Should fail the request after retries all streams");
    } catch (IndividualRequestTimeoutException e) {
        long timeoutMs = e.timeout().inMilliseconds();
        assertTrue(timeoutMs >= (10 + 20) && timeoutMs < 5000000);
    }
    writer.close();
}
 

@Test(timeout = 60000)
public void testCloseTwice() throws Exception {
    String streamName = testName.getMethodName();
    StreamImpl s = createUnstartedStream(service, streamName);

    int numWrites = 10;
    List<Future<WriteResponse>> futureList = new ArrayList<Future<WriteResponse>>(numWrites);
    for (int i = 0; i < numWrites; i++) {
        WriteOp op = createWriteOp(service, streamName, i);
        s.submit(op);
        futureList.add(op.result());
    }
    assertEquals(numWrites, s.numPendingOps());

    Future<Void> closeFuture0 = s.requestClose("close 0");
    assertTrue("Stream " + streamName + " should be set to " + StreamStatus.CLOSING,
            StreamStatus.CLOSING == s.getStatus()
                || StreamStatus.CLOSED == s.getStatus());
    Future<Void> closeFuture1 = s.requestClose("close 1");
    assertTrue("Stream " + streamName + " should be set to " + StreamStatus.CLOSING,
            StreamStatus.CLOSING == s.getStatus()
                || StreamStatus.CLOSED == s.getStatus());

    Await.result(closeFuture0);
    assertEquals("Stream " + streamName + " should be set to " + StreamStatus.CLOSED,
            StreamStatus.CLOSED, s.getStatus());
    Await.result(closeFuture1);
    assertEquals("Stream " + streamName + " should be set to " + StreamStatus.CLOSED,
            StreamStatus.CLOSED, s.getStatus());

    for (int i = 0; i < numWrites; i++) {
        Future<WriteResponse> future = futureList.get(i);
        WriteResponse wr = Await.result(future);
        assertEquals("Pending op should fail with " + StatusCode.STREAM_UNAVAILABLE,
                StatusCode.STREAM_UNAVAILABLE, wr.getHeader().getCode());
    }
}
 

@Test(timeout = 60000)
public void testWriteOpChecksumBadData() throws Exception {
    DistributedLogServiceImpl localService = createConfiguredLocalService();
    ByteBuffer buffer = getTestDataBuffer();
    WriteContext ctx = new WriteContext().setCrc32(
        ProtocolUtils.writeOpCRC32("test", buffer.array()));

    // Overwrite 1 byte to corrupt data.
    buffer.put(1, (byte)0xab);
    Future<WriteResponse> result = localService.writeWithContext("test", buffer, ctx);
    WriteResponse resp = Await.result(result);
    assertEquals(StatusCode.CHECKSUM_FAILED, resp.getHeader().getCode());
    localService.shutdown();
}
 

@Test(timeout = 60000)
public void testForwardScanControlRecord() throws Exception {
    String streamName = runtime.getMethodName();
    BKDistributedLogManager bkdlm = (BKDistributedLogManager) createNewDLM(conf, streamName);
    DLMTestUtil.generateLogSegmentNonPartitioned(bkdlm, 5 /* control recs */, 5, 1 /* txid */);

    DLSN dlsn = new DLSN(1,3,0);
    Future<LogRecordWithDLSN> futureLogrec = getFirstGreaterThanRecord(bkdlm, 0, dlsn);
    LogRecordWithDLSN logrec = Await.result(futureLogrec);
    assertEquals(new DLSN(1,5,0), logrec.getDlsn());
    bkdlm.close();
}
 

@Test(timeout = 60000)
public void testTruncateOpNoChecksum() throws Exception {
    DistributedLogServiceImpl localService = createConfiguredLocalService();
    WriteContext ctx = new WriteContext();
    Future<WriteResponse> result = localService.truncate("test", new DLSN(1,2,3).serialize(), ctx);
    WriteResponse resp = Await.result(result);
    assertEquals(StatusCode.SUCCESS, resp.getHeader().getCode());
    localService.shutdown();
}
 

private static StreamCandidate checkStream(
        final com.twitter.distributedlog.DistributedLogManagerFactory factory,
        final String streamName,
        final ExecutorService executorService,
        final BookKeeperClient bkc,
        String digestpw) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (segments.isEmpty()) {
            return null;
        }
        List<Future<LogSegmentCandidate>> futures =
                new ArrayList<Future<LogSegmentCandidate>>(segments.size());
        for (LogSegmentMetadata segment : segments) {
            futures.add(checkLogSegment(streamName, segment, executorService, bkc, digestpw));
        }
        List<LogSegmentCandidate> segmentCandidates;
        try {
            segmentCandidates = Await.result(Future.collect(futures));
        } catch (Exception e) {
            throw new IOException("Failed on checking stream " + streamName, e);
        }
        StreamCandidate streamCandidate = new StreamCandidate(streamName);
        for (LogSegmentCandidate segmentCandidate: segmentCandidates) {
            if (null != segmentCandidate) {
                streamCandidate.addLogSegmentCandidate(segmentCandidate);
            }
        }
        if (streamCandidate.segmentCandidates.isEmpty()) {
            return null;
        }
        return streamCandidate;
    } finally {
        dlm.close();
    }
}
 

@Test(timeout = 60000)
public void testStoreMaxLogSegmentSequenceNumberBadVersion() throws Exception {
    Transaction<Object> updateTxn = lsmStore.transaction();
    Versioned<Long> value = new Versioned<Long>(999L, new ZkVersion(10));
    final Promise<Version> result = new Promise<Version>();
    lsmStore.storeMaxLogSegmentSequenceNumber(updateTxn, rootZkPath, value,
            new Transaction.OpListener<Version>() {
                @Override
                public void onCommit(Version r) {
                    result.setValue(r);
                }

                @Override
                public void onAbort(Throwable t) {
                    result.setException(t);
                }
            });
    try {
        FutureUtils.result(updateTxn.execute());
        fail("Should fail on storing log segment sequence number if providing bad version");
    } catch (ZKException zke) {
        assertEquals(KeeperException.Code.BADVERSION, zke.getKeeperExceptionCode());
    }
    try {
        Await.result(result);
        fail("Should fail on storing log segment sequence number if providing bad version");
    } catch (KeeperException ke) {
        assertEquals(KeeperException.Code.BADVERSION, ke.code());
    }
    Stat stat = new Stat();
    byte[] data = zkc.get().getData(rootZkPath, false, stat);
    assertEquals(0, stat.getVersion());
    assertEquals(0, data.length);
}
 

public void markEndOfStream() throws IOException {
    try {
        Await.result(logWriter.markEndOfStream());
    } catch (IOException ioe) {
        throw ioe;
    } catch (Exception ex) {
        throw new UnexpectedException("Mark end of stream hit unexpected exception", ex);
    }
}
 

@Test(timeout = 60000)
public void testBulkWriteTotalFailureLostLock() throws Exception {
    String name = String.format("dlserver-bulk-write-%s", "lost-lock");

    dlClient.routingService.addHost(name, dlServer.getAddress());

    final int writeCount = 8;
    List<ByteBuffer> writes = new ArrayList<ByteBuffer>(writeCount + 1);
    ByteBuffer buf = ByteBuffer.allocate(8);
    writes.add(buf);
    for (long i = 1; i <= writeCount; i++) {
        writes.add(ByteBuffer.wrap(("" + i).getBytes()));
    }
    // Warm it up with a write.
    Await.result(dlClient.dlClient.write(name, ByteBuffer.allocate(8)));

    // Failpoint a lost lock, make sure the failure gets promoted to an operation failure.
    DistributedLogServiceImpl svcImpl = (DistributedLogServiceImpl) dlServer.dlServer.getLeft();
    try {
        FailpointUtils.setFailpoint(
            FailpointUtils.FailPointName.FP_WriteInternalLostLock,
            FailpointUtils.FailPointActions.FailPointAction_Default
        );
        Future<BulkWriteResponse> futures = svcImpl.writeBulkWithContext(name, writes, new WriteContext());
        assertEquals(StatusCode.LOCKING_EXCEPTION, Await.result(futures).header.code);
    } finally {
        FailpointUtils.removeFailpoint(
            FailpointUtils.FailPointName.FP_WriteInternalLostLock
        );
    }
}
 

@Test(timeout = 60000)
public void testStreamOpChecksumBadChecksum() throws Exception {
    DistributedLogServiceImpl localService = createConfiguredLocalService();
    WriteContext ctx = new WriteContext().setCrc32(999);
    Future<WriteResponse> result = localService.heartbeat("test", ctx);
    WriteResponse resp = Await.result(result);
    assertEquals(StatusCode.CHECKSUM_FAILED, resp.getHeader().getCode());
    result = localService.release("test", ctx);
    resp = Await.result(result);
    assertEquals(StatusCode.CHECKSUM_FAILED, resp.getHeader().getCode());
    result = localService.delete("test", ctx);
    resp = Await.result(result);
    assertEquals(StatusCode.CHECKSUM_FAILED, resp.getHeader().getCode());
    localService.shutdown();
}
 

@Test(timeout = 60000)
public void testWriteOpNoChecksum() throws Exception {
    DistributedLogServiceImpl localService = createConfiguredLocalService();
    WriteContext ctx = new WriteContext();
    Future<WriteResponse> result = localService.writeWithContext("test", getTestDataBuffer(), ctx);
    WriteResponse resp = Await.result(result);
    assertEquals(StatusCode.SUCCESS, resp.getHeader().getCode());
    localService.shutdown();
}
 

@Test(timeout = 60000)
public void testSubscriptionsStore() throws Exception {
    String name = "distrlog-subscriptions-store";
    String subscriber0 = "subscriber-0";
    String subscriber1 = "subscriber-1";
    String subscriber2 = "subscriber-2";

    DLSN commitPosition0 = new DLSN(4, 33, 5);
    DLSN commitPosition1 = new DLSN(4, 34, 5);
    DLSN commitPosition2 = new DLSN(5, 34, 5);
    DLSN commitPosition3 = new DLSN(6, 35, 6);

    DistributedLogManager dlm = createNewDLM(conf, name);

    SubscriptionsStore store = dlm.getSubscriptionsStore();

    // no data
    assertEquals(Await.result(store.getLastCommitPosition(subscriber0)), DLSN.NonInclusiveLowerBound);
    assertEquals(Await.result(store.getLastCommitPosition(subscriber1)), DLSN.NonInclusiveLowerBound);
    assertEquals(Await.result(store.getLastCommitPosition(subscriber2)), DLSN.NonInclusiveLowerBound);
    // empty
    assertTrue(Await.result(store.getLastCommitPositions()).isEmpty());

    // subscriber 0 advance
    Await.result(store.advanceCommitPosition(subscriber0, commitPosition0));
    assertEquals(commitPosition0, Await.result(store.getLastCommitPosition(subscriber0)));
    Map<String, DLSN> committedPositions = Await.result(store.getLastCommitPositions());
    assertEquals(1, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));

    // subscriber 1 advance
    Await.result(store.advanceCommitPosition(subscriber1, commitPosition1));
    assertEquals(commitPosition1, Await.result(store.getLastCommitPosition(subscriber1)));
    committedPositions = Await.result(store.getLastCommitPositions());
    assertEquals(2, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));

    // subscriber 2 advance
    Await.result(store.advanceCommitPosition(subscriber2, commitPosition2));
    assertEquals(commitPosition2, Await.result(store.getLastCommitPosition(subscriber2)));
    committedPositions = Await.result(store.getLastCommitPositions());
    assertEquals(3, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));
    assertEquals(commitPosition2, committedPositions.get(subscriber2));

    // subscriber 2 advance again
    DistributedLogManager newDLM = createNewDLM(conf, name);
    SubscriptionsStore newStore = newDLM.getSubscriptionsStore();
    Await.result(newStore.advanceCommitPosition(subscriber2, commitPosition3));
    newStore.close();
    newDLM.close();

    committedPositions = Await.result(store.getLastCommitPositions());
    assertEquals(3, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));
    assertEquals(commitPosition3, committedPositions.get(subscriber2));

    dlm.close();

}
 

@Test(timeout = 60000)
public void testAcquireStreamsWhenExceedMaxCachedPartitions() throws Exception {
    String streamName = testName.getMethodName() + "_0000";

    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(dlConf);
    confLocal.setMaxCachedPartitionsPerProxy(1);

    ServerConfiguration serverConfLocal = new ServerConfiguration();
    serverConfLocal.addConfiguration(serverConf);
    serverConfLocal.setStreamPartitionConverterClass(DelimiterStreamPartitionConverter.class);

    DistributedLogServiceImpl serviceLocal = createService(serverConfLocal, confLocal);
    Stream stream = serviceLocal.getLogWriter(streamName);

    // stream is cached
    assertNotNull(stream);
    assertEquals(1, serviceLocal.getStreamManager().numCached());

    // create write ops
    WriteOp op0 = createWriteOp(service, streamName, 0L);
    stream.submit(op0);
    WriteResponse wr0 = Await.result(op0.result());
    assertEquals("Op 0 should succeed",
            StatusCode.SUCCESS, wr0.getHeader().getCode());
    assertEquals(1, serviceLocal.getStreamManager().numAcquired());

    // should fail to acquire another partition
    try {
        serviceLocal.getLogWriter(testName.getMethodName() + "_0001");
        fail("Should fail to acquire new streams");
    } catch (StreamUnavailableException sue) {
        // expected
    }
    assertEquals(1, serviceLocal.getStreamManager().numCached());
    assertEquals(1, serviceLocal.getStreamManager().numAcquired());

    // should be able to acquire partitions from other streams
    String anotherStreamName = testName.getMethodName() + "-another_0001";
    Stream anotherStream = serviceLocal.getLogWriter(anotherStreamName);
    assertNotNull(anotherStream);
    assertEquals(2, serviceLocal.getStreamManager().numCached());

    // create write ops
    WriteOp op1 = createWriteOp(service, anotherStreamName, 0L);
    anotherStream.submit(op1);
    WriteResponse wr1 = Await.result(op1.result());
    assertEquals("Op 1 should succeed",
            StatusCode.SUCCESS, wr1.getHeader().getCode());
    assertEquals(2, serviceLocal.getStreamManager().numAcquired());
}
 

public Response index(String content) throws Exception {
  Future<Response> response = get().handleClusterIndexRequest(content);
  return timeout == 0 ? Await.result(response) : Await.result(response, duration);
}
 

@Test(timeout = 60000)
public void testAcquireStreams() throws Exception {
    String streamName = testName.getMethodName();
    StreamImpl s0 = createUnstartedStream(service, streamName);
    ServerConfiguration serverConf1 = new ServerConfiguration();
    serverConf1.addConfiguration(serverConf);
    serverConf1.setServerPort(9999);
    DistributedLogServiceImpl service1 = createService(serverConf1, dlConf);
    StreamImpl s1 = createUnstartedStream(service1, streamName);

    // create write ops
    WriteOp op0 = createWriteOp(service, streamName, 0L);
    s0.submit(op0);

    WriteOp op1 = createWriteOp(service1, streamName, 1L);
    s1.submit(op1);

    // check pending size
    assertEquals("Write Op 0 should be pending in service 0",
            1, s0.numPendingOps());
    assertEquals("Write Op 1 should be pending in service 1",
            1, s1.numPendingOps());

    // start acquiring s0
    s0.start();
    WriteResponse wr0 = Await.result(op0.result());
    assertEquals("Op 0 should succeed",
            StatusCode.SUCCESS, wr0.getHeader().getCode());
    assertEquals("Service 0 should acquire stream",
            StreamStatus.INITIALIZED, s0.getStatus());
    assertNotNull(s0.getManager());
    assertNotNull(s0.getWriter());
    assertNull(s0.getLastException());

    // start acquiring s1
    s1.start();
    WriteResponse wr1 = Await.result(op1.result());
    assertEquals("Op 1 should fail",
            StatusCode.FOUND, wr1.getHeader().getCode());
    // the stream will be set to ERROR and then be closed.
    assertTrue("Service 1 should be in unavailable state",
            StreamStatus.isUnavailable(s1.getStatus()));
    assertNotNull(s1.getManager());
    assertNull(s1.getWriter());
    assertNotNull(s1.getLastException());
    assertTrue(s1.getLastException() instanceof OwnershipAcquireFailedException);

    service1.shutdown();
}
 

@Benchmark
public Void ensureConst() throws Exception {
  return Await.result(constVoidFuture.ensure(ensureF));
}