org.apache.distributedlog.api.namespace.NamespaceBuilder Java Examples

private void createLogPathTest(String logName) throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    ensureURICreated(zooKeeperClient.get(), uri);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(false);
    Namespace namespace = NamespaceBuilder.newBuilder()
        .conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(logName);
    LogWriter writer;
    try {
        writer = dlm.startLogSegmentNonPartitioned();
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
        writer.commit();
        fail("Should fail to write data if stream doesn't exist.");
    } catch (IOException ioe) {
        // expected
    }
    dlm.close();
}
 

private void initDlogMeta(String dlNamespace, String un, String streamName) throws Exception {
    URI uri = createDLMURI(dlNamespace);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(true);
    newConf.setZkAclId(un);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(streamName);
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (int i = 0; i < 10; i++) {
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
    }
    writer.close();
    dlm.close();
    namespace.close();
}
 

@Before
public void before() throws Exception {
  Map<String, Object> config = new HashMap<>();
  config.put(DlogStorage.NS_URI_KEY, ROOT_URI);

  mockNamespace = mock(Namespace.class);
  mockNsBuilder = mock(NamespaceBuilder.class);
  when(mockNsBuilder.clientId(anyString())).thenReturn(mockNsBuilder);
  when(mockNsBuilder.conf(any(DistributedLogConfiguration.class))).thenReturn(mockNsBuilder);
  when(mockNsBuilder.uri(any(URI.class))).thenReturn(mockNsBuilder);
  when(mockNsBuilder.build()).thenReturn(mockNamespace);

  dlogStorage = new DlogStorage(() -> mockNsBuilder);
  dlogStorage.init(StatefulStorageTestContext.TOPOLOGY_NAME, config);
  dlogStorage = spy(dlogStorage);

  instance = StatefulStorageTestContext.getInstance();
  checkpointPartition = StatefulStorageTestContext.getInstanceStateCheckpoint();
}
 

protected void run(String[] args) throws Exception {
    logger.info("Parsing arguments for benchmark : {}", args);
    // parse command line
    parseCommandLine(args);
    statsProvider.start(conf);
    // run the benchmark
    StatsLogger statsLogger = statsProvider.getStatsLogger("dl");
    Namespace namespace =
            NamespaceBuilder.newBuilder()
                    .conf(conf)
                    .uri(uri)
                    .statsLogger(statsLogger)
                    .build();
    try {
        benchmark(namespace, streamName, statsProvider.getStatsLogger("benchmark"));
    } finally {
        namespace.close();
        statsProvider.stop();
    }
}
 

@Test(timeout = 60000)
public void testCreateLogMetadataWithCustomMetadata() throws Exception {
    String logName = testName.getMethodName();
    String logIdentifier = "<default>";
    List<String> pathsToDelete = Lists.newArrayList();

    DLMetadata.create(new BKDLConfig(zkServers, "/ledgers")).update(uri);

    Namespace namespace = NamespaceBuilder.newBuilder()
        .conf(new DistributedLogConfiguration())
        .uri(uri)
        .build();

    MetadataAccessor accessor = namespace.getNamespaceDriver().getMetadataAccessor(logName);
    accessor.createOrUpdateMetadata(logName.getBytes("UTF-8"));
    accessor.close();

    testCreateLogMetadataWithMissingPaths(uri, logName, logIdentifier, pathsToDelete, true, false);
}
 

@Override
public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException {
    config = mapper.readValue(configurationFile, Config.class);

    PropertiesConfiguration propsConf = new PropertiesConfiguration();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    try {
        propsConf.load(new StringReader(config.dlogConf));
        conf.loadConf(propsConf);
    } catch (ConfigurationException e) {
        log.error("Failed to load dlog configuration : \n{}\n", config.dlogConf, e);
        throw new IOException("Failed to load configuration : \n" + config.dlogConf + "\n", e);
    }
    URI dlogUri = URI.create(config.dlogUri);

    dlshade.org.apache.bookkeeper.stats.StatsLogger dlStatsLogger =
        new CachingStatsLogger(new StatsLoggerAdaptor(statsLogger.scope("dlog")));

    namespace = NamespaceBuilder.newBuilder()
        .conf(conf)
        .uri(dlogUri)
        .statsLogger(dlStatsLogger)
        .build();

    log.info("Initialized distributedlog namespace at {}", dlogUri);
}
 

@Test(timeout = 60000)
public void testInvalidStreamFromInvalidZkPath() throws Exception {
    String baseName = testNames.getMethodName();
    String streamName = "\0blah";
    URI uri = createDLMURI("/" + baseName);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();

    DistributedLogManager dlm = null;
    AsyncLogWriter writer = null;
    try {
        dlm = namespace.openLog(streamName);
        writer = dlm.startAsyncLogSegmentNonPartitioned();
        fail("should have thrown");
    } catch (InvalidStreamNameException e) {
    } finally {
        if (null != writer) {
            Utils.close(writer);
        }
        if (null != dlm) {
            dlm.close();
        }
        namespace.close();
    }
}
 

@Test(timeout = 60000, expected = NullPointerException.class)
public void testNullClientId() throws Exception {
    NamespaceBuilder.newBuilder()
            .conf(new DistributedLogConfiguration())
            .uri(new URI("distributedlog-bk://localhost/distributedlog"))
            .clientId(null)
            .build();
}
 

@Test(timeout = 60000, expected = NullPointerException.class)
public void testMissingSchemeInUri() throws Exception {
    NamespaceBuilder.newBuilder()
            .conf(new DistributedLogConfiguration())
            .uri(new URI("/test"))
            .build();
}
 

@Test(timeout = 60000, expected = IllegalArgumentException.class)
public void testInvalidSchemeInUri() throws Exception {
    NamespaceBuilder.newBuilder()
            .conf(new DistributedLogConfiguration())
            .uri(new URI("dist://invalid/scheme/in/uri"))
            .build();
}
 

@Test(timeout = 60000, expected = IllegalArgumentException.class)
public void testUnknownBackendInUri() throws Exception {
    NamespaceBuilder.newBuilder()
            .conf(new DistributedLogConfiguration())
            .uri(new URI("distributedlog-unknown://invalid/scheme/in/uri"))
            .build();
}
 

@Test(timeout = 60000, expected = NullPointerException.class)
public void testNullStatsLogger() throws Exception {
    NamespaceBuilder.newBuilder()
            .conf(new DistributedLogConfiguration())
            .uri(new URI("distributedlog-bk://localhost/distributedlog"))
            .statsLogger(null)
            .build();
}
 

@Test(timeout = 60000, expected = IllegalArgumentException.class)
public void testInvalidSchemeCorrectBackendInUri() throws Exception {
    NamespaceBuilder.newBuilder()
            .conf(new DistributedLogConfiguration())
            .uri(new URI("dist-bk://invalid/scheme/in/uri"))
            .build();
}
 

@Test(timeout = 60000)
public void testBuildBKDistributedLogNamespace() throws Exception {
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(new DistributedLogConfiguration())
            .uri(new URI("distributedlog-bk://" + zkServers + DLOG_NAMESPACE + "/bknamespace"))
            .build();
    try {
        assertTrue("distributedlog-bk:// should build bookkeeper based distributedlog namespace",
                namespace instanceof BKDistributedLogNamespace);
    } finally {
        namespace.close();
    }
}
 

@Test(timeout = 60000)
public void testBuildWhenMissingBackendInUri() throws Exception {
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(new DistributedLogConfiguration())
            .uri(new URI("distributedlog://" + zkServers + DLOG_NAMESPACE + "/defaultnamespace"))
            .build();
    try {
        assertTrue("distributedlog:// should build bookkeeper based distributedlog namespace",
                namespace instanceof BKDistributedLogNamespace);
    } finally {
        namespace.close();
    }
}
 

@Override
public void initialize(URI name, Configuration conf) throws IOException {
    super.initialize(name, conf);
    setConf(conf);

    // initialize

    this.rootUri = name;
    // load the configuration
    String dlConfLocation = conf.get(DLFS_CONF_FILE);
    if (null != dlConfLocation) {
        try {
            this.dlConf.loadConf(new File(dlConfLocation).toURI().toURL());
            log.info("Loaded the distributedlog configuration from {}", dlConfLocation);
        } catch (ConfigurationException e) {
            log.error("Failed to load the distributedlog configuration from " + dlConfLocation, e);
            throw new IOException("Failed to load distributedlog configuration from " + dlConfLocation);
        }
    }
    log.info("Initializing the filesystem at {}", name);
    // initialize the namespace
    this.namespace = NamespaceBuilder.newBuilder()
            .clientId("dlfs-client-" + InetAddress.getLocalHost().getHostName())
            .conf(dlConf)
            .regionId(DistributedLogConstants.LOCAL_REGION_ID)
            .uri(name)
            .build();
    log.info("Initialized the filesystem at {}", name);
}
 

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }

    String dlUriStr = args[0];
    final String streamName = args[1];

    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();

    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);

    // get the last record
    LogRecordWithDLSN lastRecord;
    DLSN dlsn;
    try {
        lastRecord = dlm.getLastLogRecord();
        dlsn = lastRecord.getDlsn();
        readLoop(dlm, dlsn);
    } catch (LogNotFoundException lnfe) {
        System.err.println("Log stream " + streamName + " is not found. Please create it first.");
        return;
    } catch (LogEmptyException lee) {
        System.err.println("Log stream " + streamName + " is empty.");
        dlsn = DLSN.InitialDLSN;
        readLoop(dlm, dlsn);
    } finally {
        dlm.close();
        namespace.close();
    }
}
 

public static void main(String[] args) throws Exception {
    if (3 != args.length) {
        System.out.println(HELP);
        return;
    }

    String dlUriStr = args[0];
    final String streamName = args[1];
    final int rewindSeconds = Integer.parseInt(args[2]);

    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();

    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);

    try {
        readLoop(dlm, rewindSeconds);
    } finally {
        dlm.close();
        namespace.close();
    }
}
 

@Before
public void setUp() throws Exception {
  config = mock(Config.class);
  when(config.getStringValue(eq(DLContext.DL_TOPOLOGIES_NS_URI)))
      .thenReturn(DL_URI);
  nsBuilder = mock(NamespaceBuilder.class);
  when(nsBuilder.clientId(anyString())).thenReturn(nsBuilder);
  when(nsBuilder.conf(any(DistributedLogConfiguration.class))).thenReturn(nsBuilder);
  when(nsBuilder.uri(any(URI.class))).thenReturn(nsBuilder);

  copier = mock(Copier.class);
  uploader = new DLUploader(() -> nsBuilder, copier);
}
 

private static Namespace openNamespace(URI uri) throws IOException {
  DistributedLogConfiguration distributedLogConfiguration = new DistributedLogConfiguration();
  distributedLogConfiguration.addProperty("bkc.allowShadedLedgerManagerFactoryClass", true);
  return NamespaceBuilder.newBuilder()
      .uri(uri)
      .clientId("dlog-util")
      .conf(distributedLogConfiguration)
      .build();
}
 

void runMonitor(DistributedLogConfiguration conf, URI dlUri) throws IOException {
    // stats
    statsProvider.getStatsLogger("monitor").registerGauge("num_streams", numOfStreamsGauge);
    logger.info("Construct dl namespace @ {}", dlUri);
    dlNamespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(dlUri)
            .build();
    if (watchNamespaceChanges) {
        dlNamespace.registerNamespaceListener(this);
    } else {
        onStreamsChanged(dlNamespace.getLogs());
    }
}
 

protected Namespace getNamespace() throws IOException {
    if (null == this.namespace) {
        this.namespace = NamespaceBuilder.newBuilder()
                .uri(getUri())
                .conf(getConf())
                .build();
    }
    return this.namespace;
}
 

/**
 * Calculating stream space usage from given <i>uri</i>.
 *
 * @param uri dl uri
 * @throws IOException
 */
public Map<String, Long> calculateStreamSpaceUsage(final URI uri) throws IOException {
    logger.info("Collecting stream space usage for {}.", uri);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();
    try {
        return calculateStreamSpaceUsage(uri, namespace);
    } finally {
        namespace.close();
    }
}
 

@Test(timeout = 60000)
public void testCheckLogExists() throws Exception {
    String name = "distrlog-check-log-exists";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    writer.flush();
    writer.commit();
    writer.close();
    dlm.close();

    URI uri = createDLMURI("/" + name);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    assertTrue(namespace.logExists(name));
    assertFalse(namespace.logExists("non-existent-log"));
    URI nonExistentUri = createDLMURI("/" + "non-existent-ns");
    Namespace nonExistentNS = NamespaceBuilder.newBuilder()
            .conf(conf).uri(nonExistentUri).build();
    assertFalse(nonExistentNS.logExists(name));

    int logCount = 0;
    Iterator<String> logIter = namespace.getLogs();
    while (logIter.hasNext()) {
        String log = logIter.next();
        logCount++;
        assertEquals(name, log);
    }
    assertEquals(1, logCount);

    namespace.close();
}
 

static MetadataAccessor createNewMetadataAccessor(DistributedLogConfiguration conf,
                                                  String name,
                                                  URI uri) throws Exception {
    // TODO: Metadata Accessor seems to be a legacy object which only used by kestrel
    //       (we might consider deprecating this)
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    return namespace.getNamespaceDriver().getMetadataAccessor(name);
}
 

public static DistributedLogManager createNewDLM(String name,
                                                 DistributedLogConfiguration conf,
                                                 URI uri) throws Exception {
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    return namespace.openLog(name);
}
 

@Test(timeout = 60000)
public void testCreateIfNotExists() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    ensureURICreated(zooKeeperClient.get(), uri);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(false);
    String streamName = "test-stream";
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(streamName);
    LogWriter writer;
    try {
        writer = dlm.startLogSegmentNonPartitioned();
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
        fail("Should fail to write data if stream doesn't exist.");
    } catch (IOException ioe) {
        // expected
    }
    dlm.close();

    // create the stream
    namespace.createLog(streamName);

    DistributedLogManager newDLM = namespace.openLog(streamName);
    LogWriter newWriter = newDLM.startLogSegmentNonPartitioned();
    newWriter.write(DLMTestUtil.getLogRecordInstance(1L));
    newWriter.close();
    newDLM.close();
}
 

@Test(timeout = 60000)
public void testCompleteLogSegmentConflicts() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();

    namespace.createLog(streamName);
    DistributedLogManager dlm1 = namespace.openLog(streamName);
    DistributedLogManager dlm2 = namespace.openLog(streamName);

    // dlm1 is writing
    BKSyncLogWriter out1 = (BKSyncLogWriter) dlm1.startLogSegmentNonPartitioned();
    out1.write(DLMTestUtil.getLogRecordInstance(1));
    // before out1 complete, out2 is in on recovery
    // it completed the log segments which bump the version of /ledgers znode
    BKAsyncLogWriter out2 = (BKAsyncLogWriter) dlm2.startAsyncLogSegmentNonPartitioned();

    try {
        out1.closeAndComplete();
        fail("Should fail closeAndComplete since other people already completed it.");
    } catch (IOException ioe) {
    }
}
 

/**
 * Create Log Segment for an pre-create stream. No max ledger sequence number recorded.
 */
@Test(timeout = 60000)
public void testCreateLogSegmentOnPrecreatedStream() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();

    namespace.createLog(streamName);
    MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber());
    DistributedLogManager dlm = namespace.openLog(streamName);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }
    MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(3, max2.getSequenceNumber());
    dlm.close();
    namespace.close();
}
 

public static void main(String[] args) throws Exception {
    if (args.length < 3) {
        System.err.println("WriterTest <uri> <stream> <num_records>");
        return;
    }

    URI uri = URI.create(args[0]);
    String streamName = args[1];
    int numRecords = Integer.parseInt(args[2]);

    DistributedLogConfiguration conf = new DistributedLogConfiguration()
        .setOutputBufferSize(0)
        .setPeriodicFlushFrequencyMilliSeconds(2);

    Namespace namespace = NamespaceBuilder.newBuilder()
        .uri(uri)
        .conf(conf)
        .build();
    try {
        try (DistributedLogManager manager = namespace.openLog(streamName)) {
            AsyncLogWriter writer = FutureUtils.result(manager.openAsyncLogWriter());
            try {
                long txid = writer.getLastTxId();
                if (txid < 0L) {
                    txid = 0L;
                }

                System.out.println("Publishing " + numRecords + " records to stream " + streamName + " .");

                for (int i = 1; i <= numRecords; ++i) {
                    String content = "record-" + (txid + i);
                    LogRecord record = new LogRecord(txid + i, content.getBytes(UTF_8));
                    FutureUtils.result(writer.write(record));
                    System.out.println("Write record : " + content);
                }

                System.out.println("Successfully published " + numRecords
                    + " records to stream " + streamName + " .");
            } finally {
                Utils.close(writer);
            }
        }
    } finally {
        namespace.close();
    }
}