org.apache.bookkeeper.client.api.DigestType Java Examples

The following examples show how to use org.apache.bookkeeper.client.api.DigestType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BookkeeperCommitLog.java    From herddb with Apache License 2.0 6 votes vote down vote up
private CommitFileWriter() throws LogNotAvailableException {
    try {
        Map<String, byte[]> metadata = new HashMap<>();
        metadata.put("tablespaceuuid", tableSpaceUUID.getBytes(StandardCharsets.UTF_8));
        metadata.put("tablespacename", tableSpaceName.getBytes(StandardCharsets.UTF_8));
        metadata.put("leader", localNodeId.getBytes(StandardCharsets.UTF_8));
        metadata.put("application", "herddb".getBytes(StandardCharsets.UTF_8));
        metadata.put("component", "commitlog".getBytes(StandardCharsets.UTF_8));
        this.out = FutureUtils.result(bookKeeper.
                newCreateLedgerOp()
                .withEnsembleSize(ensemble)
                .withWriteQuorumSize(writeQuorumSize)
                .withAckQuorumSize(ackQuorumSize)
                .withDigestType(DigestType.CRC32C)
                .withPassword(SHARED_SECRET.getBytes(StandardCharsets.UTF_8))
                .withCustomMetadata(metadata)
                .execute(), BKException.HANDLER);
        this.ledgerId = this.out.getId();
        LOGGER.log(Level.INFO, "{0} created ledger {1} (" + ensemble + "/" + writeQuorumSize + "/" + ackQuorumSize + ") bookies: {2}",
                new Object[]{tableSpaceDescription(), ledgerId, this.out.getLedgerMetadata().getAllEnsembles()});
        lastLedgerId = ledgerId;
        lastSequenceNumber.set(-1);
    } catch (BKException err) {
        throw new LogNotAvailableException(err);
    }
}
 
Example #2
Source File: BkHelperLiveTest.java    From tutorials with MIT License 6 votes vote down vote up
@Test
void whenWriteAndReadEntries_thenSuccess() throws Exception {
    LedgerHandle lh = createLedger(bk, "myledger", ledgerPassword);
    long start = System.currentTimeMillis();
    for (int i = 0; i < 1000; i++) {
        byte[] data = new String("message-" + i).getBytes();
        lh.append(data);
    }
    lh.close();
    long elapsed = System.currentTimeMillis() - start;
    LOG.info("Entries added to ledgerId " + lh.getId() + ", elapsed=" + elapsed);

    Long ledgerId = findLedgerByName(bk, "myledger").orElse(null);
    assertNotNull(ledgerId);
    lh = bk.openLedger(ledgerId, BookKeeper.DigestType.MAC, ledgerPassword);
    long lastId = lh.readLastConfirmed();
    Enumeration<LedgerEntry> entries = lh.readEntries(0, lastId);
    while (entries.hasMoreElements()) {
        LedgerEntry entry = entries.nextElement();
        String msg = new String(entry.getEntry());
        LOG.info("Entry: id=" + entry.getEntryId() + ", data=" + msg);
    }
}
 
Example #3
Source File: BkHelperLiveTest.java    From tutorials with MIT License 6 votes vote down vote up
@Test
void whenWriteEntriesAsync_thenSuccess() throws Exception {
    CompletableFuture<Object> f = bk.newCreateLedgerOp()
      .withDigestType(DigestType.MAC)
      .withPassword(ledgerPassword)
      .execute()
      .thenApply((wh) -> {
          List<CompletableFuture<Long>> ops = new ArrayList<>();
          for (int i = 0; i < 1000; i++) {
              byte[] data = String.format("message-%04d", i)
                .getBytes();
              ops.add(wh.appendAsync(data));
        }
        return CompletableFuture.allOf(ops.stream()
          .toArray(CompletableFuture[]::new))
          .thenCompose((v) -> wh.closeAsync());
    });
    f.get(5, TimeUnit.MINUTES);
}
 
Example #4
Source File: ManagedLedgerErrorsTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void digestError() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger",
            new ManagedLedgerConfig().setDigestType(DigestType.CRC32));
    ledger.openCursor("c1");
    ledger.addEntry("entry".getBytes());

    ledger.close();

    try {
        ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setDigestType(DigestType.MAC));
        fail("should fail for digest error");
    } catch (ManagedLedgerException e) {
        // ok
    }
}
 
Example #5
Source File: FileSystemManagedLedgerOffloaderTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
private ReadHandle buildReadHandle() throws Exception {

        lh = bk.createLedger(1,1,1, BookKeeper.DigestType.CRC32, "foobar".getBytes());

        int i = 0;
        int blocksWritten = 1;
        while (blocksWritten <= numberOfEntries) {
            byte[] entry = ("foobar"+i).getBytes();
            blocksWritten++;
            lh.addEntry(entry);
            i++;
        }
        lh.close();

        return bk.newOpenLedgerOp().withLedgerId(lh.getId())
                .withPassword("foobar".getBytes()).withDigestType(DigestType.CRC32).execute().get();
    }
 
Example #6
Source File: OffloadIndexTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
private LedgerMetadata createLedgerMetadata() throws Exception {

        Map<String, byte[]> metadataCustom = Maps.newHashMap();
        metadataCustom.put("key1", "value1".getBytes(UTF_8));
        metadataCustom.put("key7", "value7".getBytes(UTF_8));

        ArrayList<BookieSocketAddress> bookies = Lists.newArrayList();
        BookieSocketAddress BOOKIE1 = new BookieSocketAddress("127.0.0.1:3181");
        BookieSocketAddress BOOKIE2 = new BookieSocketAddress("127.0.0.2:3181");
        BookieSocketAddress BOOKIE3 = new BookieSocketAddress("127.0.0.3:3181");
        bookies.add(0, BOOKIE1);
        bookies.add(1, BOOKIE2);
        bookies.add(2, BOOKIE3);

        return LedgerMetadataBuilder.create().withEnsembleSize(3).withWriteQuorumSize(3).withAckQuorumSize(2)
            .withDigestType(DigestType.CRC32C).withPassword("password".getBytes(UTF_8))
            .withCustomMetadata(metadataCustom).withClosedState().withLastEntryId(5000).withLength(100)
            .newEnsembleEntry(0L, bookies).build();
    }
 
Example #7
Source File: BlobStoreManagedLedgerOffloaderTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private ReadHandle buildReadHandle(int maxBlockSize, int blockCount) throws Exception {
    Assert.assertTrue(maxBlockSize > DataBlockHeaderImpl.getDataStartOffset());

    LedgerHandle lh = bk.createLedger(1,1,1, BookKeeper.DigestType.CRC32, "foobar".getBytes());

    int i = 0;
    int bytesWrittenCurrentBlock = DataBlockHeaderImpl.getDataStartOffset();
    int blocksWritten = 1;
    int entries = 0;

    while (blocksWritten < blockCount
           || bytesWrittenCurrentBlock < maxBlockSize/2) {
        byte[] entry = ("foobar"+i).getBytes();
        int sizeInBlock = entry.length + 12 /* ENTRY_HEADER_SIZE */;

        if (bytesWrittenCurrentBlock + sizeInBlock > maxBlockSize) {
            bytesWrittenCurrentBlock = DataBlockHeaderImpl.getDataStartOffset();
            blocksWritten++;
            entries = 0;
        }
        entries++;

        lh.addEntry(entry);
        bytesWrittenCurrentBlock += sizeInBlock;
        i++;
    }

    lh.close();

    return bk.newOpenLedgerOp().withLedgerId(lh.getId())
        .withPassword("foobar".getBytes()).withDigestType(DigestType.CRC32).execute().get();
}
 
Example #8
Source File: BookKeeperConfig.java    From pravega with Apache License 2.0 5 votes vote down vote up
static DigestType getDigestType(String digestType) {
    if (digestType.equals(DigestType.MAC.name())) {
        return DigestType.MAC;
    } else if (digestType.equals(DigestType.CRC32.name())) {
        return DigestType.CRC32;
    } else if (digestType.equals(DigestType.DUMMY.name())) {
        return DigestType.DUMMY;
    } else {
        // Default digest for performance reasons.
        return DigestType.CRC32C;
    }
}
 
Example #9
Source File: BkHelperLiveTest.java    From tutorials with MIT License 5 votes vote down vote up
@Test
void whenCreateLedgerAsync_thenSuccess() throws Exception {

    CompletableFuture<WriteHandle> cf = bk.newCreateLedgerOp()
      .withDigestType(org.apache.bookkeeper.client.api.DigestType.MAC)
      .withPassword("password".getBytes())
      .execute();

    WriteHandle handle = cf.get(1, TimeUnit.MINUTES);
    assertNotNull(handle);
    handle.close();

}
 
Example #10
Source File: BkHelperLiveTest.java    From tutorials with MIT License 5 votes vote down vote up
@Test
void whenCreateLedger_thenSuccess() throws Exception {
    LedgerHandle lh = bk.createLedger(BookKeeper.DigestType.MAC, ledgerPassword);
    assertNotNull(lh);
    assertNotNull(lh.getId());
    LOG.info("[I33] Ledge created: id=" + lh.getId());
}
 
Example #11
Source File: ManagedLedgerBkTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testChangeCrcType() throws Exception {
    ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
    config.setDigestType(DigestType.CRC32);
    ManagedLedger ledger = factory.open("my_test_ledger", config);
    ManagedCursor c1 = ledger.openCursor("c1");

    ledger.addEntry("entry-0".getBytes());
    ledger.addEntry("entry-1".getBytes());
    ledger.addEntry("entry-2".getBytes());

    ledger.close();

    config.setDigestType(DigestType.CRC32C);
    ledger = factory.open("my_test_ledger", config);
    c1 = ledger.openCursor("c1");

    ledger.addEntry("entry-3".getBytes());

    assertEquals(c1.getNumberOfEntries(), 4);
    assertEquals(c1.getNumberOfEntriesInBacklog(false), 4);

    List<Entry> entries = c1.readEntries(4);
    assertEquals(entries.size(), 4);
    for (int i = 0; i < 4; i++) {
        assertEquals(new String(entries.get(i).getData()), "entry-" + i);
    }

    factory.shutdown();
}
 
Example #12
Source File: ManagedLedgerBkTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testOfflineTopicBacklog() throws Exception {
    ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
    factoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc, factoryConf);
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1)
            .setMetadataAckQuorumSize(1);
    ManagedLedger ledger = factory.open("property/cluster/namespace/my-ledger", config);
    ManagedCursor cursor = ledger.openCursor("c1");

    int N = 1;

    for (int i = 0; i < N; i++) {
        String entry = "entry-" + i;
        ledger.addEntry(entry.getBytes());
    }

    List<Entry> entries = cursor.readEntries(N);
    assertEquals(N, entries.size());
    entries.forEach(e -> e.release());
    ledger.close();

    ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(
            DigestType.CRC32, "".getBytes(Charsets.UTF_8), "", false);
    PersistentOfflineTopicStats offlineTopicStats = offlineTopicBacklog.getEstimatedUnloadedTopicBacklog(
            (ManagedLedgerFactoryImpl) factory, "property/cluster/namespace/my-ledger");
    factory.shutdown();
    assertNotNull(offlineTopicStats);
}
 
Example #13
Source File: ManagedLedgerBkTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
/**
 * When auto-replication is triggered, if there were no writes on the ML during the grace period, auto-replication
 * will close the ledger an re-replicate it. After that, the next write will get a FencedException. We should
 * recover from this condition by creating a new ledger and retrying the write.
 */
@Test
public void ledgerFencedByAutoReplication() throws Exception {
    ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", config);
    ManagedCursor c1 = ledger.openCursor("c1");

    PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes());

    // Trigger the closure of the data ledger
    bkc.openLedger(p1.getLedgerId(), BookKeeper.DigestType.CRC32C, new byte[] {});

    ledger.addEntry("entry-2".getBytes());

    assertEquals(2, c1.getNumberOfEntries());
    assertEquals(2, c1.getNumberOfEntriesInBacklog(false));

    PositionImpl p3 = (PositionImpl) ledger.addEntry("entry-3".getBytes());

    // Now entry-2 should have been written before entry-3
    assertEquals(3, c1.getNumberOfEntries());
    assertEquals(3, c1.getNumberOfEntriesInBacklog(false));
    assertTrue(p1.getLedgerId() != p3.getLedgerId());
    factory.shutdown();
}
 
Example #14
Source File: BookKeeperConfigTest.java    From pravega with Apache License 2.0 5 votes vote down vote up
@Test
public void testDefaultValues() {
    BookKeeperConfig cfg = BookKeeperConfig.builder()
            .build();
    Assert.assertEquals("localhost:2181", cfg.getZkAddress());
    Assert.assertEquals(Duration.ofMillis(10000), cfg.getZkSessionTimeout());
    Assert.assertEquals(Duration.ofMillis(10000), cfg.getZkConnectionTimeout());
    Assert.assertEquals("/segmentstore/containers", cfg.getZkMetadataPath());
    Assert.assertEquals(2, cfg.getZkHierarchyDepth());
    Assert.assertEquals(5, cfg.getMaxWriteAttempts());
    Assert.assertEquals(3, cfg.getBkEnsembleSize());
    Assert.assertEquals(2, cfg.getBkAckQuorumSize());
    Assert.assertEquals(3, cfg.getBkWriteQuorumSize());
    Assert.assertEquals(60000, cfg.getBkWriteTimeoutMillis());
    Assert.assertEquals(30000, cfg.getBkReadTimeoutMillis());
    Assert.assertEquals(64, cfg.getBkReadBatchSize());
    Assert.assertEquals(256 * 1024 * 1024, cfg.getMaxOutstandingBytes());
    Assert.assertEquals(1024 * 1024 * 1024, cfg.getBkLedgerMaxSize());
    Assert.assertEquals(0, cfg.getBKPassword().length);
    Assert.assertEquals("", cfg.getBkLedgerPath());
    Assert.assertEquals(false, cfg.isTLSEnabled());
    Assert.assertEquals("config/client.truststore.jks", cfg.getTlsTrustStore());
    Assert.assertEquals("", cfg.getTlsTrustStorePasswordPath());
    Assert.assertEquals(false, cfg.isEnforceMinNumRacksPerWriteQuorum());
    Assert.assertEquals(2, cfg.getMinNumRacksPerWriteQuorum());
    Assert.assertEquals("/opt/pravega/scripts/sample-bookkeeper-topology.sh", cfg.getNetworkTopologyFileName());
    Assert.assertEquals(DigestType.CRC32C, cfg.getDigestType());
}
 
Example #15
Source File: ManagedLedgerOfflineBacklog.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public ManagedLedgerOfflineBacklog(DigestType digestType, byte[] password, String brokerName,
        boolean accurate) {
    this.digestType = BookKeeper.DigestType.fromApiDigestType(digestType);
    this.password = password;
    this.accurate = accurate;
    this.brokerName = brokerName;
}
 
Example #16
Source File: BookKeeperConfigTest.java    From pravega with Apache License 2.0 5 votes vote down vote up
@Test
public void testDigestTypeConversion() {
    Assert.assertEquals(DigestType.CRC32, BookKeeperConfig.getDigestType("CRC32"));
    Assert.assertEquals(DigestType.CRC32C, BookKeeperConfig.getDigestType("CRC32C"));
    Assert.assertEquals(DigestType.DUMMY, BookKeeperConfig.getDigestType("DUMMY"));
    Assert.assertEquals(DigestType.MAC, BookKeeperConfig.getDigestType("MAC"));
    Assert.assertEquals(DigestType.CRC32C, BookKeeperConfig.getDigestType("any-other-value"));
}
 
Example #17
Source File: OffloadPrefixReadTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Override
public DigestType getDigestType() { return digestType; }
 
Example #18
Source File: ManagedLedgerConfig.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * @param digestType
 *            the digestType to set
 */
public ManagedLedgerConfig setDigestType(DigestType digestType) {
    this.digestType = digestType;
    return this;
}
 
Example #19
Source File: ManagedLedgerConfig.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * @return the digestType
 */
public DigestType getDigestType() {
    return digestType;
}
 
Example #20
Source File: OffloadUtils.java    From pulsar with Apache License 2.0 4 votes vote down vote up
public static LedgerMetadata parseLedgerMetadata(byte[] bytes) throws IOException {
    DataFormats.LedgerMetadataFormat ledgerMetadataFormat = DataFormats.LedgerMetadataFormat.newBuilder().mergeFrom(bytes).build();
    LedgerMetadataBuilder builder = LedgerMetadataBuilder.create()
            .withLastEntryId(ledgerMetadataFormat.getLastEntryId())
            .withPassword(ledgerMetadataFormat.getPassword().toByteArray())
            .withClosedState()
            .withMetadataFormatVersion(2)
            .withLength(ledgerMetadataFormat.getLength())
            .withAckQuorumSize(ledgerMetadataFormat.getAckQuorumSize())
            .withCreationTime(ledgerMetadataFormat.getCtime())
            .withWriteQuorumSize(ledgerMetadataFormat.getQuorumSize())
            .withEnsembleSize(ledgerMetadataFormat.getEnsembleSize());
    ledgerMetadataFormat.getSegmentList().forEach(segment -> {
        ArrayList<BookieSocketAddress> addressArrayList = new ArrayList<>();
        segment.getEnsembleMemberList().forEach(address -> {
            try {
                addressArrayList.add(new BookieSocketAddress(address));
            } catch (IOException e) {
                log.error("Exception when create BookieSocketAddress. ", e);
            }
        });
        builder.newEnsembleEntry(segment.getFirstEntryId(), addressArrayList);
    });

    if (ledgerMetadataFormat.getCustomMetadataCount() > 0) {
        Map<String, byte[]> customMetadata = Maps.newHashMap();
        ledgerMetadataFormat.getCustomMetadataList().forEach(
                entry -> customMetadata.put(entry.getKey(), entry.getValue().toByteArray()));
        builder.withCustomMetadata(customMetadata);
    }

    switch (ledgerMetadataFormat.getDigestType()) {
        case HMAC:
            builder.withDigestType(DigestType.MAC);
            break;
        case CRC32:
            builder.withDigestType(DigestType.CRC32);
            break;
        case CRC32C:
            builder.withDigestType(DigestType.CRC32C);
            break;
        case DUMMY:
            builder.withDigestType(DigestType.DUMMY);
            break;
        default:
            throw new IllegalArgumentException("Unable to convert digest type " + ledgerMetadataFormat.getDigestType());
    }

    return builder.build();
}
 
Example #21
Source File: PulsarConfigurationLoaderTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test
public void testPulsarConfiguraitonLoadingStream() throws Exception {
    File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
    if (testConfigFile.exists()) {
        testConfigFile.delete();
    }
    final String zkServer = "z1.example.com,z2.example.com,z3.example.com";
    PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)));
    printWriter.println("zookeeperServers=" + zkServer);
    printWriter.println("configurationStoreServers=gz1.example.com,gz2.example.com,gz3.example.com/foo");
    printWriter.println("brokerDeleteInactiveTopicsEnabled=true");
    printWriter.println("statusFilePath=/tmp/status.html");
    printWriter.println("managedLedgerDefaultEnsembleSize=1");
    printWriter.println("backlogQuotaDefaultLimitGB=18");
    printWriter.println("clusterName=usc");
    printWriter.println("brokerClientAuthenticationPlugin=test.xyz.client.auth.plugin");
    printWriter.println("brokerClientAuthenticationParameters=role:my-role");
    printWriter.println("superUserRoles=appid1,appid2");
    printWriter.println("brokerServicePort=7777");
    printWriter.println("brokerServicePortTls=8777");
    printWriter.println("webServicePort=");
    printWriter.println("webServicePortTls=");
    printWriter.println("managedLedgerDefaultMarkDeleteRateLimit=5.0");
    printWriter.println("managedLedgerDigestType=CRC32C");
    printWriter.println("managedLedgerCacheSizeMB=");
    printWriter.println("bookkeeperDiskWeightBasedPlacementEnabled=true");
    printWriter.close();
    testConfigFile.deleteOnExit();
    InputStream stream = new FileInputStream(testConfigFile);
    final ServiceConfiguration serviceConfig = PulsarConfigurationLoader.create(stream, ServiceConfiguration.class);
    assertNotNull(serviceConfig);
    assertEquals(serviceConfig.getZookeeperServers(), zkServer);
    assertEquals(serviceConfig.isBrokerDeleteInactiveTopicsEnabled(), true);
    assertEquals(serviceConfig.getBacklogQuotaDefaultLimitGB(), 18);
    assertEquals(serviceConfig.getClusterName(), "usc");
    assertEquals(serviceConfig.getBrokerClientAuthenticationParameters(), "role:my-role");
    assertEquals(serviceConfig.getBrokerServicePort().get(), new Integer(7777));
    assertEquals(serviceConfig.getBrokerServicePortTls().get(), new Integer(8777));
    assertFalse(serviceConfig.getWebServicePort().isPresent());
    assertFalse(serviceConfig.getWebServicePortTls().isPresent());
    assertEquals(serviceConfig.getManagedLedgerDigestType(), DigestType.CRC32C);
    assertTrue(serviceConfig.getManagedLedgerCacheSizeMB() > 0);
    assertTrue(serviceConfig.isBookkeeperDiskWeightBasedPlacementEnabled());
}
 
Example #22
Source File: BkHelperLiveTest.java    From tutorials with MIT License 4 votes vote down vote up
@Test
void whenWriteAndReadEntriesAsync_thenSuccess() throws Exception {
    CompletableFuture<Long> f = bk.newCreateLedgerOp()
      .withDigestType(DigestType.MAC)
      .withPassword(ledgerPassword)
      .execute()
      .thenApply((wh) -> {
          List<CompletableFuture<Long>> ops = new ArrayList<>();
          for (int i = 0; i < 1000; i++) {
              byte[] data = String.format("message-%04d", i)
                .getBytes();
              ops.add(wh.appendAsync(data));
          }
          return CompletableFuture.allOf(ops.stream()
            .toArray(CompletableFuture[]::new))
            .thenCompose((v) -> wh.closeAsync())
            .thenApply((v) -> wh.getId());
      })
      .thenCompose((lf) -> lf); // flatten the futures
    Long ledgerId = f.get(5, TimeUnit.MINUTES);
    LOG.info("Ledger created with 1000 entries: ledgerId=" + ledgerId);

    // Now let's read data back...
    CompletableFuture<LedgerEntries> ef = bk.newOpenLedgerOp()
      .withLedgerId(ledgerId)
      .withPassword(ledgerPassword)
      .withDigestType(DigestType.MAC)
      .execute()
      .thenCompose((rh) -> {
          return rh.readLastAddConfirmedAsync()
            .thenCompose((lastId) -> rh.readAsync(0, lastId));
      });
    LedgerEntries entries = ef.get(5, TimeUnit.MINUTES);
    
    // Check all writes where OK
    long count = 0;
    Iterator<org.apache.bookkeeper.client.api.LedgerEntry> it = entries.iterator();
    while (it.hasNext()) {
        org.apache.bookkeeper.client.api.LedgerEntry e = it.next();
        String msg = new String(e.getEntryBytes());
        assertEquals(String.format("message-%04d", count), msg);
        count++;
    }
    assertEquals(1000, count);
    LOG.info("Got entries: count=" + count);
}
 
Example #23
Source File: KafkaServiceConfigurationTest.java    From kop with Apache License 2.0 4 votes vote down vote up
@Test
public void testConfigurationUtilsStream() throws Exception {
    File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
    if (testConfigFile.exists()) {
        testConfigFile.delete();
    }
    final String zkServer = "z1.example.com,z2.example.com,z3.example.com";
    PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)));
    printWriter.println("zookeeperServers=" + zkServer);
    printWriter.println("configurationStoreServers=gz1.example.com,gz2.example.com,gz3.example.com/foo");
    printWriter.println("brokerDeleteInactiveTopicsEnabled=true");
    printWriter.println("statusFilePath=/tmp/status.html");
    printWriter.println("managedLedgerDefaultEnsembleSize=1");
    printWriter.println("backlogQuotaDefaultLimitGB=18");
    printWriter.println("clusterName=usc");
    printWriter.println("brokerClientAuthenticationPlugin=test.xyz.client.auth.plugin");
    printWriter.println("brokerClientAuthenticationParameters=role:my-role");
    printWriter.println("superUserRoles=appid1,appid2");
    printWriter.println("brokerServicePort=7777");
    printWriter.println("brokerServicePortTls=8777");
    printWriter.println("webServicePort=");
    printWriter.println("webServicePortTls=");
    printWriter.println("managedLedgerDefaultMarkDeleteRateLimit=5.0");
    printWriter.println("managedLedgerDigestType=CRC32C");

    printWriter.close();
    testConfigFile.deleteOnExit();

    InputStream stream = new FileInputStream(testConfigFile);
    final KafkaServiceConfiguration kafkaServiceConfig =
        ConfigurationUtils.create(stream, KafkaServiceConfiguration.class);

    assertNotNull(kafkaServiceConfig);
    assertEquals(kafkaServiceConfig.getZookeeperServers(), zkServer);
    assertEquals(kafkaServiceConfig.isBrokerDeleteInactiveTopicsEnabled(), true);
    assertEquals(kafkaServiceConfig.getBacklogQuotaDefaultLimitGB(), 18);
    assertEquals(kafkaServiceConfig.getClusterName(), "usc");
    assertEquals(kafkaServiceConfig.getBrokerClientAuthenticationParameters(), "role:my-role");
    assertEquals(kafkaServiceConfig.getBrokerServicePort().get(), new Integer(7777));
    assertEquals(kafkaServiceConfig.getBrokerServicePortTls().get(), new Integer(8777));
    assertFalse(kafkaServiceConfig.getWebServicePort().isPresent());
    assertFalse(kafkaServiceConfig.getWebServicePortTls().isPresent());
    assertEquals(kafkaServiceConfig.getManagedLedgerDigestType(), DigestType.CRC32C);
}