org.apache.bookkeeper.conf.ServerConfiguration Java Examples

BookieServer newBookie() throws Exception {
  int port = nextPort++;
  ServerConfiguration bookieConf = new ServerConfiguration();
  bookieConf.setBookiePort(port);
  File tmpdir = File.createTempFile("bookie" + Integer.toString(port) + "_",
                                    "test");
  tmpdir.delete();
  tmpdir.mkdir();

  bookieConf.setZkServers(zkEnsemble);
  bookieConf.setJournalDirName(tmpdir.getPath());
  bookieConf.setLedgerDirNames(new String[] { tmpdir.getPath() });

  BookieServer b = new BookieServer(bookieConf);
  b.start();
  for (int i = 0; i < 10 && !b.isRunning(); i++) {
    Thread.sleep(10000);
  }
  if (!b.isRunning()) {
    throw new IOException("Bookie would not start");
  }
  return b;
}
 

private String startBookie(boolean format) throws Exception {
    if (format && !bookies.isEmpty()) {
        throw new Exception("cannot format, you aleady have bookies");
    }
    ServerConfiguration conf = createBookieConf(nextBookiePort++);

    if (format) {
        BookKeeperAdmin.initNewCluster(conf);
        BookKeeperAdmin.format(conf, false, true);
    }

    BookieServer bookie = new BookieServer(conf);
    bookies.add(bookie);
    bookie.start();
    return bookie.getLocalAddress().getSocketAddress().toString();
}
 

private ServerConfiguration createBookieConf(int port) {
    ServerConfiguration conf = new ServerConfiguration();
    conf.setBookiePort(port++);
    LOG.log(Level.INFO, "STARTING BOOKIE at port {0}", String.valueOf(port));
    conf.setUseHostNameAsBookieID(true);
    // no need to preallocate journal and entrylog in tests
    conf.setEntryLogFilePreAllocationEnabled(false);
    conf.setProperty("journalPreAllocSizeMB", 1);
    Path targetDir = path.resolve("bookie_data_" + conf.getBookiePort());
    conf.setMetadataServiceUri("zk+null://" + zkServer.getConnectString() + herddb.server.ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH_DEFAULT);
    conf.setLedgerDirNames(new String[]{targetDir.toAbsolutePath().toString()});
    conf.setJournalDirName(targetDir.toAbsolutePath().toString());
    conf.setFlushInterval(10000);
    conf.setGcWaitTime(5);
    conf.setJournalFlushWhenQueueEmpty(true);
    //        conf.setJournalBufferedEntriesThreshold(1);
    conf.setAutoRecoveryDaemonEnabled(false);
    // no need for real network in tests
    conf.setEnableLocalTransport(true);
    conf.setDisableServerSocketBind(true);
    // no need to fsync in tests
    conf.setJournalSyncData(false);
    conf.setAllowLoopback(true);
    conf.setProperty("journalMaxGroupWaitMSec", 10); // default 200ms
    return conf;
}
 

public void startStoppedBookie(String addr) throws Exception {
    int index = 0;
    for (BookieServer bookie : bookies) {
        if (bookie.getLocalAddress().getSocketAddress().toString().equals(addr)) {
            if (bookie.isRunning()) {
                throw new Exception("you did not stop bookie " + addr);
            }
            ServerConfiguration conf = createBookieConf(bookie.getLocalAddress().getPort());
            BookieServer newBookie = new BookieServer(conf);
            bookies.set(index, newBookie);
            newBookie.start();
            return;
        }
        index++;
    }
    throw new Exception("Cannot find bookie " + addr);
}
 

public ZKTestEnv(Path path) throws Exception {
    zkServer = new TestingServer(1282, path.toFile(), true);
    // waiting for ZK to be reachable
    CountDownLatch latch = new CountDownLatch(1);
    ZooKeeper zk = new ZooKeeper(zkServer.getConnectString(),
            herddb.server.ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT_DEFAULT, (WatchedEvent event) -> {
                LOG.log(Level.INFO, "ZK EVENT {0}", event);
                if (event.getState() == KeeperState.SyncConnected) {
                    latch.countDown();
                }
            });
    try {
        if (!latch.await(herddb.server.ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT_DEFAULT, TimeUnit.MILLISECONDS)) {
            LOG.log(Level.INFO, "ZK client did not connect withing {0} seconds, maybe the server did not start up",
                    herddb.server.ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT_DEFAULT);
        }
    } finally {
        zk.close(1000);
    }
    this.path = path;
}
 

/**
 * Helper method to startup a bookie server using a configuration object. Also, starts the auto recovery process if
 * isAutoRecoveryEnabled is true.
 *
 * @param conf
 *            Server Configuration Object
 *
 */
protected BookieServer startBookie(ServerConfiguration conf, String ledgerRootPath) throws Exception {
    BookieServer server = new BookieServer(conf);
    bsConfs.add(conf);
    bs.add(server);

    server.start();

    if (bkc == null) {
        bkc = new BookKeeperTestClient(baseClientConf);
    }

    int port = conf.getBookiePort();
    while (bkc.getZkHandle()
        .exists(ledgerRootPath + "/available/" + InetAddress.getLocalHost().getHostAddress() + ":" + port,
            false) == null) {
        Thread.sleep(500);
    }

    bkc.readBookiesBlocking();
    LOG.info("New bookie on port " + port + " has been created.");

    return server;
}
 

BookieServer newBookie() throws Exception {
  int port = nextPort++;
  ServerConfiguration bookieConf = new ServerConfiguration();
  bookieConf.setBookiePort(port);
  File tmpdir = File.createTempFile("bookie" + Integer.toString(port) + "_",
                                    "test");
  tmpdir.delete();
  tmpdir.mkdir();

  bookieConf.setZkServers(zkEnsemble);
  bookieConf.setJournalDirName(tmpdir.getPath());
  bookieConf.setLedgerDirNames(new String[] { tmpdir.getPath() });

  BookieServer b = new BookieServer(bookieConf);
  b.start();
  for (int i = 0; i < 10 && !b.isRunning(); i++) {
    Thread.sleep(10000);
  }
  if (!b.isRunning()) {
    throw new IOException("Bookie would not start");
  }
  return b;
}
 

/**
 * Restart bookie servers using new configuration settings. Also restart the respective auto recovery process, if
 * isAutoRecoveryEnabled is true.
 *
 * @param newConf
 *            New Configuration Settings
 * @throws InterruptedException
 * @throws IOException
 * @throws KeeperException
 * @throws BookieException
 */
public void restartBookies(ServerConfiguration newConf) throws Exception {
    // shut down bookie server
    for (BookieServer server : bs) {
        server.shutdown();
        stopAutoRecoveryService(server);
    }
    bs.clear();
    Thread.sleep(1000);
    // restart them to ensure we can't

    List<ServerConfiguration> bsConfsCopy = new ArrayList<ServerConfiguration>(bsConfs);
    bsConfs.clear();
    for (ServerConfiguration conf : bsConfsCopy) {
        if (null != newConf) {
            conf.loadConf(newConf);
        }
        startBookie(conf);
    }
}
 

private LocalDLMEmulator(final int numBookies, final boolean shouldStartZK,
                         final String zkHost, final int zkPort, final int initialBookiePort,
                         final int zkTimeoutSec, final ServerConfiguration serverConf) throws Exception {
    this.numBookies = numBookies;
    this.zkHost = zkHost;
    this.zkPort = zkPort;
    this.zkEnsemble = zkHost + ":" + zkPort;
    this.uri = URI.create("distributedlog://" + zkEnsemble + DLOG_NAMESPACE);
    this.zkTimeoutSec = zkTimeoutSec;
    this.bkStartupThread = new Thread() {
        public void run() {
            try {
                LOG.info("Starting {} bookies : allowLoopback = {}", numBookies, serverConf.getAllowLoopback());
                LocalBookKeeper.startLocalBookies(zkHost, zkPort,
                        numBookies, shouldStartZK, initialBookiePort, serverConf);
                LOG.info("{} bookies are started.");
            } catch (InterruptedException e) {
                // go away quietly
            } catch (Exception e) {
                LOG.error("Error starting local bk", e);
            }
        }
    };
}
 

/**
 * Kill a bookie by its socket address. Also, stops the autorecovery process for the corresponding bookie server, if
 * isAutoRecoveryEnabled is true.
 *
 * @param addr
 *            Socket Address
 * @return the configuration of killed bookie
 * @throws InterruptedException
 */
public ServerConfiguration killBookie(InetSocketAddress addr) throws Exception {
    BookieServer toRemove = null;
    int toRemoveIndex = 0;
    for (BookieServer server : bs) {
        if (server.getLocalAddress().equals(addr)) {
            server.shutdown();
            toRemove = server;
            break;
        }
        ++toRemoveIndex;
    }
    if (toRemove != null) {
        stopAutoRecoveryService(toRemove);
        bs.remove(toRemove);
        return bsConfs.remove(toRemoveIndex);
    }
    return null;
}
 

protected ServerConfiguration newServerConfiguration(int port, String zkServers, File journalDir,
        File[] ledgerDirs, String ledgerRootPath) {
    ServerConfiguration conf = new ServerConfiguration(baseConf);
    conf.setBookiePort(port);
    if (ledgerRootPath != "") {
        conf.setMetadataServiceUri("zk://" + zkUtil.getZooKeeperConnectString() + ledgerRootPath);
    }else {
        conf.setZkServers(zkServers);
    }
    conf.setJournalDirName(journalDir.getPath());
    conf.setAllowLoopback(true);
    conf.setFlushInterval(60 * 1000);
    conf.setGcWaitTime(60 * 1000);
    conf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap);
    String[] ledgerDirNames = new String[ledgerDirs.length];
    for (int i = 0; i < ledgerDirs.length; i++) {
        ledgerDirNames[i] = ledgerDirs[i].getPath();
    }
    conf.setLedgerDirNames(ledgerDirNames);
    conf.setLedgerStorageClass("org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage");
    conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 4);
    conf.setProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB, 4);
    return conf;
}
 

private LocalDLMEmulator(final int numBookies, final boolean shouldStartZK, final String zkHost, final int zkPort, final int initialBookiePort, final int zkTimeoutSec, final ServerConfiguration serverConf) throws Exception {
    this.numBookies = numBookies;
    this.zkHost = zkHost;
    this.zkPort = zkPort;
    this.zkEnsemble = zkHost + ":" + zkPort;
    this.uri = URI.create("distributedlog://" + zkEnsemble + DLOG_NAMESPACE);
    this.zkTimeoutSec = zkTimeoutSec;
    this.bkStartupThread = new Thread() {
        public void run() {
            try {
                LOG.info("Starting {} bookies : allowLoopback = {}", numBookies, serverConf.getAllowLoopback());
                LocalBookKeeper.startLocalBookies(zkHost, zkPort, numBookies, shouldStartZK, initialBookiePort, serverConf);
                LOG.info("{} bookies are started.");
            } catch (InterruptedException e) {
                // go away quietly
            } catch (Exception e) {
                LOG.error("Error starting local bk", e);
            }
        }
    };
}
 

/**
 * Start cluster. Also, starts the auto recovery process for each bookie, if isAutoRecoveryEnabled is true.
 *
 * @throws Exception
 */
protected void startBKCluster(String ledgerPath) throws Exception {
    baseClientConf.setMetadataServiceUri("zk://" + zkUtil.getZooKeeperConnectString() + ledgerPath + "/ledgers");
    baseClientConf.setUseV2WireProtocol(true);
    baseClientConf.setEnableDigestTypeAutodetection(true);
    baseClientConf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap);
    if (numBookies > 0) {
        bkc = new BookKeeperTestClient(baseClientConf);
    }

    // Create Bookie Servers (B1, B2, B3)
    for (int i = 0; i < numBookies; i++) {
        if (ledgerPath != "") {
            ServerConfiguration configuration = newServerConfiguration(ledgerPath + "/ledgers");
            startBookie(configuration, ledgerPath + "/ledgers");
        }else {
            startNewBookie();
        }
    }
}
 

@BeforeClass
protected void setup() throws Exception {
    super.setup();

    // Start bookies with specific racks
    for (int i = 0; i < NUM_BOOKIES; i++) {
        File bkDataDir = Files.createTempDirectory("bk" + Integer.toString(i) + "test").toFile();
        ServerConfiguration conf = new ServerConfiguration();

        conf.setBookiePort(0);
        conf.setZkServers("127.0.0.1:" + bkEnsemble.getZookeeperPort());
        conf.setJournalDirName(bkDataDir.getPath());
        conf.setLedgerDirNames(new String[] { bkDataDir.getPath() });
        conf.setAllowLoopback(true);

        // Use different advertised addresses for each bookie, so we can place them in different
        // racks.
        // Eg: 1st bookie will be 10.0.0.1, 2nd 10.0.0.2 and so on
        String addr = String.format("10.0.0.%d", i + 1);
        conf.setAdvertisedAddress(addr);

        BookieServer bs = new BookieServer(conf, NullStatsLogger.INSTANCE);

        bs.start();
        bookies.add(bs);
    }

}
 

private DistributedLogCluster(DistributedLogConfiguration dlConf,
                              ServerConfiguration bkConf,
                              int numBookies,
                              boolean shouldStartZK,
                              String zkServers,
                              int zkPort,
                              boolean shouldStartProxy,
                              int proxyPort) throws Exception {
    this.dlConf = dlConf;
    if (shouldStartZK) {
        File zkTmpDir = IOUtils.createTempDir("zookeeper", "distrlog");
        tmpDirs.add(zkTmpDir);
        if (0 == zkPort) {
            Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkTmpDir);
            this.zks = serverAndPort.getLeft();
            zkPort = serverAndPort.getRight();
        } else {
            this.zks = LocalBookKeeper.runZookeeper(1000, zkPort, zkTmpDir);
        }
    } else {
        this.zks = null;
    }
    this.dlmEmulator = LocalDLMEmulator.newBuilder()
            .numBookies(numBookies)
            .zkHost(zkServers)
            .zkPort(zkPort)
            .serverConf(bkConf)
            .shouldStartZK(false)
            .build();
    this.shouldStartProxy = shouldStartProxy;
    this.proxyPort = proxyPort;
}
 

public void start(boolean enableStreamStorage) throws  Exception {
    LOG.debug("Local ZK/BK starting ...");
    ServerConfiguration conf = new ServerConfiguration();
    // Use minimal configuration requiring less memory for unit tests
    conf.setLedgerStorageClass(DbLedgerStorage.class.getName());
    conf.setProperty("dbStorage_writeCacheMaxSizeMb", 2);
    conf.setProperty("dbStorage_readAheadCacheMaxSizeMb", 1);
    conf.setProperty("dbStorage_rocksDB_writeBufferSizeMB", 1);
    conf.setProperty("dbStorage_rocksDB_blockCacheSize", 1024 * 1024);
    conf.setFlushInterval(60000);
    conf.setJournalSyncData(false);
    conf.setProperty("journalMaxGroupWaitMSec", 0L);
    conf.setAllowLoopback(true);
    conf.setGcWaitTime(60000);
    conf.setNumAddWorkerThreads(0);
    conf.setNumReadWorkerThreads(0);
    conf.setNumHighPriorityWorkerThreads(0);
    conf.setNumJournalCallbackThreads(0);
    conf.setServerNumIOThreads(1);
    conf.setNumLongPollWorkerThreads(1);
    conf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap);

    runZookeeper(1000);
    initializeZookeper();
    runBookies(conf);

    if (enableStreamStorage) {
        runStreamStorage(new CompositeConfiguration());
    }
}
 

public void startStandalone(ServerConfiguration conf, boolean enableStreamStorage) throws Exception {
    LOG.debug("Local ZK/BK starting ...");
    conf.setAdvertisedAddress(advertisedAddress);

    runZookeeper(1000);
    initializeZookeper();
    runBookies(conf);
    if (enableStreamStorage) {
        runStreamStorage(new CompositeConfiguration());
    }
}
 

protected ServerConfiguration newServerConfiguration(String ledgerRootPath) throws Exception {
    File f = File.createTempFile("bookie", "test");
    tmpDirs.add(f);
    f.delete();
    f.mkdir();
    
    int port = 0;
    return newServerConfiguration(port, zkUtil.getZooKeeperConnectString(), f, new File[] { f }, ledgerRootPath);
}
 

/**
 * Kill a bookie by index. Also, stops the respective auto recovery process for this bookie, if
 * isAutoRecoveryEnabled is true.
 *
 * @param index
 *            Bookie Index
 * @return the configuration of killed bookie
 * @throws InterruptedException
 * @throws IOException
 */
public ServerConfiguration killBookie(int index) throws Exception {
    if (index >= bs.size()) {
        throw new IOException("Bookie does not exist");
    }
    BookieServer server = bs.get(index);
    server.shutdown();
    stopAutoRecoveryService(server);
    bs.remove(server);
    return bsConfs.remove(index);
}
 

@BeforeMethod
void setup() {
    try {
        bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
        ServerConfiguration conf = new ServerConfiguration();
        conf.setNettyMaxFrameSizeBytes(10 * 1024 * 1024);
        bkEnsemble.startStandalone(conf, false);

        configuration = new ServiceConfiguration();
        configuration.setZookeeperServers("127.0.0.1:" + bkEnsemble.getZookeeperPort());
        configuration.setAdvertisedAddress("localhost");
        configuration.setWebServicePort(Optional.of(0));
        configuration.setClusterName("max_message_test");
        configuration.setBrokerServicePort(Optional.of(0));
        configuration.setAuthorizationEnabled(false);
        configuration.setAuthenticationEnabled(false);
        configuration.setManagedLedgerMaxEntriesPerLedger(5);
        configuration.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
        configuration.setMaxMessageSize(10 * 1024 * 1024);

        pulsar = new PulsarService(configuration);
        pulsar.start();

        String url = "http://127.0.0.1:" + pulsar.getListenPortHTTP().get();
        admin = PulsarAdmin.builder().serviceHttpUrl(url).build();
        admin.clusters().createCluster("max_message_test", new ClusterData(url));
        admin.tenants()
             .createTenant("test", new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet("max_message_test")));
        admin.namespaces().createNamespace("test/message", Sets.newHashSet("max_message_test"));
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

protected DLServer(DistributedLogConfiguration dlConf, URI uri, int basePort) throws Exception {
    proxyPort = basePort;

    boolean success = false;
    int retries = 0;
    Pair<DistributedLogServiceImpl, Server> serverPair = null;
    while (!success) {
        try {
            com.twitter.distributedlog.service.config.ServerConfiguration serverConf =
                    new com.twitter.distributedlog.service.config.ServerConfiguration();
            serverConf.loadConf(dlConf);
            serverConf.setServerShardId(proxyPort);
            serverPair = DistributedLogServer.runServer(
                    serverConf,
                    dlConf,
                    uri,
                    new IdentityStreamPartitionConverter(),
                    new NullStatsProvider(),
                    proxyPort);
            success = true;
        } catch (BindException be) {
            retries++;
            if (retries > MAX_RETRIES) {
                throw be;
            }
            proxyPort++;
            if (proxyPort > MAX_PORT) {
                proxyPort = MIN_PORT;
            }
        }
    }

    LOG.info("Runnning DL on port {}", proxyPort);

    dlServer = serverPair;
    address = DLSocketAddress.getSocketAddress(proxyPort);
}
 

public static ServerConfiguration loadTestBkConf() {
    ServerConfiguration conf = new ServerConfiguration();
    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
    URL confUrl = classLoader.getResource("bk_server.conf");
    try {
        if (null != confUrl) {
            conf.loadConf(confUrl);
            LOG.info("loaded bk_server.conf from resources");
        }
    } catch (org.apache.commons.configuration.ConfigurationException ex) {
        LOG.warn("loading conf failed", ex);
    }
    conf.setAllowLoopback(true);
    return conf;
}
 

public BookieServer newBookie() throws Exception {
    ServerConfiguration bookieConf = new ServerConfiguration();
    bookieConf.setZkTimeout(zkTimeoutSec * 1000);
    bookieConf.setBookiePort(0);
    bookieConf.setAllowLoopback(true);
    File tmpdir = File.createTempFile("bookie" + UUID.randomUUID() + "_",
        "test");
    if (!tmpdir.delete()) {
        LOG.debug("Fail to delete tmpdir " + tmpdir);
    }
    if (!tmpdir.mkdir()) {
        throw new IOException("Fail to create tmpdir " + tmpdir);
    }
    tmpDirs.add(tmpdir);

    bookieConf.setZkServers(zkEnsemble);
    bookieConf.setJournalDirName(tmpdir.getPath());
    bookieConf.setLedgerDirNames(new String[]{tmpdir.getPath()});

    BookieServer b = new BookieServer(bookieConf);
    b.start();
    for (int i = 0; i < 10 && !b.isRunning(); i++) {
        Thread.sleep(10000);
    }
    if (!b.isRunning()) {
        throw new IOException("Bookie would not start");
    }
    return b;
}
 

private BookieServer runBookie(int bkPort) throws Exception {
    // Attempt to reuse an existing data directory. This is useful in case of stops & restarts, when we want to preserve
    // already committed data.
    File journalDir = this.journalDirs.getOrDefault(bkPort, null);
    if (journalDir == null) {
        journalDir = IOUtils.createTempDir("bookiejournal_" + bkPort, "_test");
        log.info("Journal Dir[{}]: {}.", bkPort, journalDir.getPath());
        this.journalDirs.put(bkPort, journalDir);
        setupTempDir(journalDir);
    }

    File ledgerDir = this.ledgerDirs.getOrDefault(bkPort, null);
    if (ledgerDir == null) {
        ledgerDir = Strings.isNullOrEmpty(this.ledgersDir) ? null : new File(this.ledgersDir);
        ledgerDir = IOUtils.createTempDir("bookieledger_" + bkPort, "_test", ledgerDir);
        log.info("Ledgers Dir[{}]: {}.", bkPort, ledgerDir.getPath());
        this.ledgerDirs.put(bkPort, ledgerDir);
        setupTempDir(ledgerDir);
    }

    val conf = new ServerConfiguration();
    conf.setBookiePort(bkPort);
    conf.setMetadataServiceUri("zk://" + LOOPBACK_ADDRESS.getHostAddress() + ":" + this.zkPort + ledgersPath);
    conf.setJournalDirName(journalDir.getPath());
    conf.setLedgerDirNames(new String[]{ledgerDir.getPath()});
    conf.setAllowLoopback(true);
    conf.setJournalAdaptiveGroupWrites(true);

    if (secureBK) {
        conf.setTLSProvider("OpenSSL");
        conf.setTLSProviderFactoryClass("org.apache.bookkeeper.tls.TLSContextFactory");
        conf.setTLSKeyStore(this.tLSKeyStore);
        conf.setTLSKeyStorePasswordPath(this.tLSKeyStorePasswordPath);
    }

    log.info("Starting Bookie at port " + bkPort);
    val bs = new BookieServer(conf);
    bs.start();
    return bs;
}
 

public static ServerConfiguration loadTestBkConf() {
    ServerConfiguration conf = new ServerConfiguration();
    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
    URL confUrl = classLoader.getResource("bk_server.conf");
    try {
        if (null != confUrl) {
            conf.loadConf(confUrl);
            LOG.info("loaded bk_server.conf from resources");
        }
    } catch (org.apache.commons.configuration.ConfigurationException ex) {
        LOG.warn("loading conf failed", ex);
    }
    conf.setAllowLoopback(true);
    return conf;
}
 

private DistributedLogCluster(DistributedLogConfiguration dlConf,
                              ServerConfiguration bkConf,
                              int numBookies,
                              boolean shouldStartZK,
                              String zkServers,
                              int zkPort,
                              boolean shouldStartProxy,
                              int proxyPort,
                              boolean thriftmux) throws Exception {
    this.dlConf = dlConf;
    if (shouldStartZK) {
        File zkTmpDir = IOUtils.createTempDir("zookeeper", "distrlog");
        tmpDirs.add(zkTmpDir);
        if (0 == zkPort) {
            Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkTmpDir);
            this.zks = serverAndPort.getLeft();
            zkPort = serverAndPort.getRight();
        } else {
            this.zks = LocalBookKeeper.runZookeeper(1000, zkPort, zkTmpDir);
        }
    } else {
        this.zks = null;
    }
    this.dlmEmulator = LocalDLMEmulator.newBuilder()
            .numBookies(numBookies)
            .zkHost(zkServers)
            .zkPort(zkPort)
            .serverConf(bkConf)
            .shouldStartZK(false)
            .build();
    this.shouldStartProxy = shouldStartProxy;
    this.proxyPort = proxyPort;
    this.thriftmux = thriftmux;
}
 

private void runBookies(ServerConfiguration baseConf) throws Exception {
    LOG.info("Starting Bookie(s)");
    // Create Bookie Servers (B1, B2, B3)

    bs = new BookieServer[numberOfBookies];
    bsConfs = new ServerConfiguration[numberOfBookies];

    for (int i = 0; i < numberOfBookies; i++) {

        File bkDataDir = isNotBlank(bkDataDirName)
                ? Files.createDirectories(Paths.get(bkDataDirName + Integer.toString(i))).toFile()
                : Files.createTempDirectory("bk" + Integer.toString(i) + "test").toFile();

        if (this.clearOldData) {
            cleanDirectory(bkDataDir);
        }

        int bookiePort = portManager.get();

        // Ensure registration Z-nodes are cleared when standalone service is restarted ungracefully
        String registrationZnode = String.format("/ledgers/available/%s:%d", baseConf.getAdvertisedAddress(), bookiePort);
        if (zkc.exists(registrationZnode, null) != null) {
            try {
                zkc.delete(registrationZnode, -1);
            } catch (NoNodeException nne) {
                // Ignore if z-node was just expired
            }
        }

        bsConfs[i] = new ServerConfiguration(baseConf);
        // override settings
        bsConfs[i].setBookiePort(bookiePort);
        bsConfs[i].setZkServers("127.0.0.1:" + zkPort);
        bsConfs[i].setJournalDirName(bkDataDir.getPath());
        bsConfs[i].setLedgerDirNames(new String[] { bkDataDir.getPath() });
        bsConfs[i].setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap);
        bsConfs[i].setAllowEphemeralPorts(true);

        try {
            bs[i] = new BookieServer(bsConfs[i], NullStatsLogger.INSTANCE);
        } catch (InvalidCookieException e) {
            // InvalidCookieException can happen if the machine IP has changed
            // Since we are running here a local bookie that is always accessed
            // from localhost, we can ignore the error
            for (String path : zkc.getChildren("/ledgers/cookies", false)) {
                zkc.delete("/ledgers/cookies/" + path, -1);
            }

            // Also clean the on-disk cookie
            new File(new File(bkDataDir, "current"), "VERSION").delete();

            // Retry to start the bookie after cleaning the old left cookie
            bs[i] = new BookieServer(bsConfs[i], NullStatsLogger.INSTANCE);
        }
        bs[i].start();
        LOG.debug("Local BK[{}] started (port: {}, data_directory: {})", i, bookiePort,
                bkDataDir.getAbsolutePath());
    }
}
 

public void start() throws Exception {

        if (config == null) {
            throw new IllegalArgumentException("Null configuration is provided");
        }

        if (config.getAdvertisedAddress() != null && !config.getListeners().contains(config.getAdvertisedAddress())) {
            String err = "Error config: advertisedAddress - " + config.getAdvertisedAddress() + " and listeners - "
                + config.getListeners() + " not match.";
            log.error(err);
            throw new IllegalArgumentException(err);
        }

        log.info("--- setup KafkaStandaloneStarter ---");

        if (!this.isOnlyBroker()) {
            ServerConfiguration bkServerConf = new ServerConfiguration();
            bkServerConf.loadConf(new File(configFile).toURI().toURL());

            // Start LocalBookKeeper
            bkEnsemble = new LocalBookkeeperEnsemble(
                    this.getNumOfBk(), this.getZkPort(), this.getBkPort(), this.getStreamStoragePort(), this.getZkDir(),
                    this.getBkDir(), this.isWipeData(), "127.0.0.1");
            bkEnsemble.startStandalone(bkServerConf, !this.isNoStreamStorage());
        }

        if (this.isNoBroker()) {
            return;
        }

        // Start Broker
        kafkaBroker = new KafkaService(config);
        kafkaBroker.start();

        URL webServiceUrl = new URL(
                String.format("http://%s:%d", config.getAdvertisedAddress(), config.getWebServicePort().get()));
        final String brokerServiceUrl = String.format("pulsar://%s:%d", config.getAdvertisedAddress(),
            config.getBrokerServicePort().get());

        admin = PulsarAdmin.builder().serviceHttpUrl(webServiceUrl.toString()).authentication(
                config.getBrokerClientAuthenticationPlugin(), config.getBrokerClientAuthenticationParameters()).build();

        final String cluster = config.getClusterName();

        createDefaultNameSpace(webServiceUrl, brokerServiceUrl, cluster);

        log.info("--- setup completed ---");
    }
 

public void startStandalone() throws Exception {
    startStandalone(new ServerConfiguration(), false);
}
 

protected ServerConfiguration newServerConfiguration() throws Exception {
    return newServerConfiguration("");
}