org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble Java Examples

@BeforeMethod
public void setup() throws Exception {
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    ServiceConfiguration config = new ServiceConfiguration();
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setWebServicePort(Optional.ofNullable(0));
    config.setClusterName("usc");
    config.setAdvertisedAddress("localhost");
    config.setBrokerServicePort(Optional.ofNullable(0));
    config.setAdvertisedAddress(advertisedAddress);
    config.setManagedLedgerMaxEntriesPerLedger(5);
    config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
    pulsar = new PulsarService(config);
    pulsar.start();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  if (!isJdkSupported)
    return;

  bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, TestUtil::nextFreePort);
  bkEnsemble.start();

  final int brokerWebServicePort = TestUtil.nextFreePort();
  final int brokerServicePort = TestUtil.nextFreePort();

  final ServiceConfiguration config = new ServiceConfiguration();
  config.setClusterName(CLUSTER_NAME);
  final Set<String> superUsers = Sets.newHashSet("superUser");
  config.setSuperUserRoles(superUsers);
  config.setWebServicePort(brokerWebServicePort);
  config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
  config.setBrokerServicePort(brokerServicePort);
  config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
  config.setTlsAllowInsecureConnection(true);
  config.setAdvertisedAddress("localhost");

  pulsarService = new PulsarService(config);
  pulsarService.start();

  try (final PulsarAdmin admin = pulsarService.getAdminClient()) {
    final ClusterData clusterData = new ClusterData(pulsarService.getBrokerServiceUrl());
    admin.clusters().createCluster(CLUSTER_NAME, clusterData);

    final TenantInfo propAdmin = new TenantInfo();
    propAdmin.getAdminRoles().add("superUser");
    propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList(CLUSTER_NAME)));

    admin.tenants().createTenant("public", propAdmin);
    admin.namespaces().createNamespace("public/default", Sets.newHashSet(CLUSTER_NAME));
  }
}
 

public void start() throws IOException {
    try {
        zks = new ZooKeeperServer(zkTmpDir, zkTmpDir, ZooKeeperServer.DEFAULT_TICK_TIME);
        zks.setMaxSessionTimeout(20000);
        serverFactory = new NIOServerCnxnFactory();
        serverFactory.configure(new InetSocketAddress(zkPort), 1000);
        serverFactory.startup(zks);
    } catch (Exception e) {
        log.error("Exception while instantiating ZooKeeper", e);
    }

    LocalBookkeeperEnsemble.waitForServerUp(hostPort, 30000);
    log.info("ZooKeeper started at {}", hostPort);
}
 

@BeforeClass
void setup() throws Exception {
    log.info("---- Initializing SLAMonitoringTest -----");
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // start brokers
    for (int i = 0; i < BROKER_COUNT; i++) {
        ServiceConfiguration config = new ServiceConfiguration();
        config.setBrokerServicePort(Optional.of(0));
        config.setClusterName("my-cluster");
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.of(0));
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setBrokerServicePort(Optional.of(0));
        config.setDefaultNumberOfNamespaceBundles(1);
        config.setLoadBalancerEnabled(false);
        configurations[i] = config;

        pulsarServices[i] = new PulsarService(config);
        pulsarServices[i].start();

        brokerWebServicePorts[i] = pulsarServices[i].getListenPortHTTP().get();
        brokerNativeBrokerPorts[i] = pulsarServices[i].getBrokerListenPort().get();
        brokerUrls[i] = new URL(pulsarServices[i].getWebServiceAddress());
        pulsarAdmins[i] = PulsarAdmin.builder().serviceHttpUrl(brokerUrls[i].toString()).build();
    }

    Thread.sleep(100);

    createTenant(pulsarAdmins[BROKER_COUNT - 1]);
    for (int i = 0; i < BROKER_COUNT; i++) {
        String topic = String.format("%s/%s/%s:%s", NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster",
                pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i]);
        pulsarAdmins[0].namespaces().createNamespace(topic);
    }
}
 

@BeforeMethod
void setup() throws Exception {
    log.info("---- Initializing TopicOwnerTest -----");
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // start brokers
    for (int i = 0; i < BROKER_COUNT; i++) {
        ServiceConfiguration config = new ServiceConfiguration();
        config.setBrokerServicePort(Optional.of(0));
        config.setClusterName("my-cluster");
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.of(0));
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setDefaultNumberOfNamespaceBundles(1);
        config.setLoadBalancerEnabled(false);
        configurations[i] = config;

        pulsarServices[i] = new PulsarService(config);
        pulsarServices[i].start();

        pulsarAdmins[i] = PulsarAdmin.builder()
                .serviceHttpUrl(pulsarServices[i].getWebServiceAddress())
                .build();
    }
    Thread.sleep(1000);
}
 

@BeforeMethod
void setup() {
    try {
        bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
        ServerConfiguration conf = new ServerConfiguration();
        conf.setNettyMaxFrameSizeBytes(10 * 1024 * 1024);
        bkEnsemble.startStandalone(conf, false);

        configuration = new ServiceConfiguration();
        configuration.setZookeeperServers("127.0.0.1:" + bkEnsemble.getZookeeperPort());
        configuration.setAdvertisedAddress("localhost");
        configuration.setWebServicePort(Optional.of(0));
        configuration.setClusterName("max_message_test");
        configuration.setBrokerServicePort(Optional.of(0));
        configuration.setAuthorizationEnabled(false);
        configuration.setAuthenticationEnabled(false);
        configuration.setManagedLedgerMaxEntriesPerLedger(5);
        configuration.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
        configuration.setMaxMessageSize(10 * 1024 * 1024);

        pulsar = new PulsarService(configuration);
        pulsar.start();

        String url = "http://127.0.0.1:" + pulsar.getListenPortHTTP().get();
        admin = PulsarAdmin.builder().serviceHttpUrl(url).build();
        admin.clusters().createCluster("max_message_test", new ClusterData(url));
        admin.tenants()
             .createTenant("test", new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet("max_message_test")));
        admin.namespaces().createNamespace("test/message", Sets.newHashSet("max_message_test"));
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

@BeforeMethod
protected void setup() throws Exception {
    try {
        // start local bookie and zookeeper
        bkEnsemble = new LocalBookkeeperEnsemble(numberOfBookies, 0, () -> 0);
        bkEnsemble.start();

        // start pulsar service
        config = new ServiceConfiguration();
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.of(0));
        config.setClusterName("usc");
        config.setBrokerServicePort(Optional.of(0));
        config.setAuthorizationEnabled(false);
        config.setAuthenticationEnabled(false);
        config.setManagedLedgerMaxEntriesPerLedger(5);
        config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
        config.setAdvertisedAddress("127.0.0.1");
        config.setAllowAutoTopicCreationType("non-partitioned");

        pulsar = new PulsarService(config);
        pulsar.start();

        admin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddress()).build();

        admin.clusters().createCluster("usc", new ClusterData(pulsar.getWebServiceAddress()));
        admin.tenants().createTenant("prop",
                new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet("usc")));
    } catch (Throwable t) {
        log.error("Error setting up broker test", t);
        Assert.fail("Broker test setup failed");
    }
}
 

@BeforeMethod
void setup() throws Exception {

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // Start broker 1
    ServiceConfiguration config1 = spy(new ServiceConfiguration());
    config1.setClusterName("use");
    config1.setWebServicePort(Optional.of(0));
    config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config1.setBrokerServicePort(Optional.of(0));
    config1.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
    config1.setBrokerServicePortTls(Optional.of(0));
    config1.setWebServicePortTls(Optional.of(0));
    config1.setAdvertisedAddress("localhost");
    pulsar1 = new PulsarService(config1);
    pulsar1.start();

    url1 = new URL(pulsar1.getWebServiceAddress());
    admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();
    brokerStatsClient1 = admin1.brokerStats();
    primaryHost = pulsar1.getWebServiceAddress();

    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setClusterName("use");
    config2.setWebServicePort(Optional.of(0));
    config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config2.setBrokerServicePort(Optional.of(0));
    config2.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
    config2.setBrokerServicePortTls(Optional.of(0));
    config2.setWebServicePortTls(Optional.of(0));
    config2.setAdvertisedAddress("localhost");
    pulsar2 = new PulsarService(config2);
    pulsar2.start();

    url2 = new URL(pulsar2.getWebServiceAddress());
    admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();
    brokerStatsClient2 = admin2.brokerStats();
    secondaryHost = pulsar2.getWebServiceAddress();
    Thread.sleep(100);
}
 

@BeforeMethod
void setup(Method method) throws Exception {

    log.info("--- Setting up method {} ---", method.getName());

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    config = spy(new ServiceConfiguration());
    config.setClusterName("use");
    Set<String> superUsers = Sets.newHashSet("superUser");
    config.setSuperUserRoles(superUsers);
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    Set<String> providers = new HashSet<>();
    providers.add(AuthenticationProviderTls.class.getName());
    config.setAuthenticationEnabled(true);
    config.setAuthorizationEnabled(true);
    config.setAuthenticationProviders(providers);
    config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config.setTlsAllowInsecureConnection(true);
    config.setAdvertisedAddress("localhost");
    functionsWorkerService = spy(createPulsarFunctionWorker(config));
    AuthenticationService authenticationService = new AuthenticationService(config);
    AuthorizationService authorizationService = new AuthorizationService(config, mock(ConfigurationCacheService.class));
    when(functionsWorkerService.getAuthenticationService()).thenReturn(authenticationService);
    when(functionsWorkerService.getAuthorizationService()).thenReturn(authorizationService);
    when(functionsWorkerService.isInitialized()).thenReturn(true);

    PulsarAdmin admin = mock(PulsarAdmin.class);
    Tenants tenants = mock(Tenants.class);
    when(admin.tenants()).thenReturn(tenants);
    when(functionsWorkerService.getBrokerAdmin()).thenReturn(admin);
    Set<String> admins = Sets.newHashSet("superUser");
    TenantInfo tenantInfo = new TenantInfo(admins, null);
    when(tenants.getTenantInfo(any())).thenReturn(tenantInfo);
    Namespaces namespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(namespaces);
    when(namespaces.getNamespaces(any())).thenReturn(namespaceList);

    // mock: once authentication passes, function should return response: function already exist
    FunctionMetaDataManager dataManager = mock(FunctionMetaDataManager.class);
    when(dataManager.containsFunction(any(), any(), any())).thenReturn(true);
    when(functionsWorkerService.getFunctionMetaDataManager()).thenReturn(dataManager);

    workerServer = new WorkerServer(functionsWorkerService);
    workerServer.start();
    Thread.sleep(2000);
    String functionTlsUrl = String.format("https://%s:%s",
            functionsWorkerService.getWorkerConfig().getWorkerHostname(), workerServer.getListenPortHTTPS().get());

    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);

    functionAdmin = PulsarAdmin.builder().serviceHttpUrl(functionTlsUrl)
            .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH).allowTlsInsecureConnection(true)
            .authentication(authTls).build();

    Thread.sleep(100);
}
 

@BeforeMethod
void setup(Method method) throws Exception {

    log.info("--- Setting up method {} ---", method.getName());

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    config = spy(new ServiceConfiguration());
    config.setClusterName("use");
    Set<String> superUsers = Sets.newHashSet("superUser");
    config.setSuperUserRoles(superUsers);
    config.setWebServicePort(Optional.of(0));
    config.setWebServicePortTls(Optional.of(0));
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerServicePort(Optional.of(0));
    config.setBrokerServicePortTls(Optional.of(0));
    config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());


    Set<String> providers = new HashSet<>();
    providers.add(AuthenticationProviderTls.class.getName());
    config.setAuthenticationEnabled(true);
    config.setAuthenticationProviders(providers);
    config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config.setTlsAllowInsecureConnection(true);


    functionsWorkerService = createPulsarFunctionWorker(config);
    Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService);
    pulsar = new PulsarService(config, functionWorkerService, (exitCode) -> {});
    pulsar.start();
    urlTls = new URL(pulsar.getBrokerServiceUrlTls());

    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);

    admin = spy(
            PulsarAdmin.builder()
                    .serviceHttpUrl(pulsar.getWebServiceAddressTls())
                    .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH)
                    .allowTlsInsecureConnection(true)
                    .authentication(authTls)
                    .build());

    brokerStatsClient = admin.brokerStats();
    primaryHost = pulsar.getWebServiceAddress();

    // update cluster metadata
    ClusterData clusterData = new ClusterData(urlTls.toString());
    admin.clusters().updateCluster(config.getClusterName(), clusterData);

    ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(this.workerConfig.getPulsarServiceUrl());
    if (isNotBlank(workerConfig.getClientAuthenticationPlugin())
            && isNotBlank(workerConfig.getClientAuthenticationParameters())) {
        clientBuilder.enableTls(workerConfig.isUseTls());
        clientBuilder.allowTlsInsecureConnection(workerConfig.isTlsAllowInsecureConnection());
        clientBuilder.authentication(workerConfig.getClientAuthenticationPlugin(),
                workerConfig.getClientAuthenticationParameters());
    }
    pulsarClient = clientBuilder.build();

    TenantInfo propAdmin = new TenantInfo();
    propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use")));
    admin.tenants().updateTenant(tenant, propAdmin);

    Thread.sleep(100);
}
 

@BeforeMethod(timeOut = 60000)
void setup(Method method) throws Exception {

    log.info("--- Setting up method {} ---", method.getName());

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    config = spy(new ServiceConfiguration());
    config.setClusterName("use");
    final Set<String> superUsers = Sets.newHashSet("superUser");
    config.setSuperUserRoles(superUsers);
    config.setWebServicePort(Optional.of(0));
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerServicePort(Optional.of(0));
    config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
    config.setAdvertisedAddress("localhost");

    functionsWorkerService = createPulsarFunctionWorker(config);
    final Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService);
    pulsar = new PulsarService(config, functionWorkerService, (exitCode) -> {});
    pulsar.start();

    admin = spy(PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddress()).build());

    brokerStatsClient = admin.brokerStats();
    primaryHost = pulsar.getWebServiceAddress();

    // update cluster metadata
    final ClusterData clusterData = new ClusterData(pulsar.getBrokerServiceUrl());
    admin.clusters().updateCluster(config.getClusterName(), clusterData);

    final ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(this.workerConfig.getPulsarServiceUrl());
    pulsarClient = clientBuilder.build();

    final TenantInfo propAdmin = new TenantInfo();
    propAdmin.getAdminRoles().add("superUser");
    propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use")));
    admin.tenants().updateTenant(tenant, propAdmin);

    Thread.sleep(100);
}
 

@BeforeMethod
void setup(Method method) throws Exception {

    // delete all function temp files
    File dir = new File(System.getProperty("java.io.tmpdir"));
    File[] foundFiles = dir.listFiles((ignoredDir, name) -> name.startsWith("function"));

    for (File file : foundFiles) {
        file.delete();
    }

    log.info("--- Setting up method {} ---", method.getName());

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    config = spy(new ServiceConfiguration());
    config.setClusterName("use");
    Set<String> superUsers = Sets.newHashSet("superUser");
    config.setSuperUserRoles(superUsers);
    config.setWebServicePort(Optional.of(0));
    config.setWebServicePortTls(Optional.of(0));
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerServicePort(Optional.of(0));
    config.setBrokerServicePortTls(Optional.of(0));
    config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
    config.setTlsAllowInsecureConnection(true);
    config.setAdvertisedAddress("localhost");

    Set<String> providers = new HashSet<>();
    providers.add(AuthenticationProviderTls.class.getName());
    config.setAuthenticationEnabled(true);
    config.setAuthenticationProviders(providers);

    config.setAuthorizationEnabled(true);
    config.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName());

    config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);

    config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
    config.setBrokerClientAuthenticationParameters(
            "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH);
    config.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
    config.setBrokerClientTlsEnabled(true);
    config.setAllowAutoTopicCreationType("non-partitioned");

    functionsWorkerService = createPulsarFunctionWorker(config);

    Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService);
    pulsar = new PulsarService(config, functionWorkerService, (exitCode) -> {});
    pulsar.start();

    String brokerServiceUrl = pulsar.getWebServiceAddressTls();
    urlTls = new URL(brokerServiceUrl);


    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);

    admin = spy(
            PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH)
                    .allowTlsInsecureConnection(true).authentication(authTls).build());

    brokerStatsClient = admin.brokerStats();
    primaryHost = pulsar.getWebServiceAddress();

    // update cluster metadata
    ClusterData clusterData = new ClusterData(urlTls.toString());
    admin.clusters().updateCluster(config.getClusterName(), clusterData);

    ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(this.workerConfig.getPulsarServiceUrl());
    if (isNotBlank(workerConfig.getClientAuthenticationPlugin())
            && isNotBlank(workerConfig.getClientAuthenticationParameters())) {
        clientBuilder.enableTls(workerConfig.isUseTls());
        clientBuilder.allowTlsInsecureConnection(workerConfig.isTlsAllowInsecureConnection());
        clientBuilder.authentication(workerConfig.getClientAuthenticationPlugin(),
                workerConfig.getClientAuthenticationParameters());
    }
    pulsarClient = clientBuilder.build();

    TenantInfo propAdmin = new TenantInfo();
    propAdmin.getAdminRoles().add("superUser");
    propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use")));
    admin.tenants().updateTenant(tenant, propAdmin);

    System.setProperty(JAVA_INSTANCE_JAR_PROPERTY,
            FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath());

    functionWorkerService.get().getLeaderService().waitLeaderInit();
}
 

@BeforeMethod
protected void setup() throws Exception {
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(4, 0, () -> 0);
    bkEnsemble.start();
}
 

@BeforeMethod
void setup() throws Exception {
    try {
        // start local bookie and zookeeper
        bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
        bkEnsemble.start();

        // start pulsar service
        config = new ServiceConfiguration();
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.ofNullable(0));
        config.setClusterName("usc");
        config.setBrokerServicePort(Optional.ofNullable(0));
        config.setAuthorizationEnabled(false);
        config.setAuthenticationEnabled(false);
        config.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
        config.setManagedLedgerMaxEntriesPerLedger(MAX_ENTRIES_PER_LEDGER);
        config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
        config.setAllowAutoTopicCreationType("non-partitioned");

        pulsar = new PulsarService(config);
        pulsar.start();

        adminUrl = new URL("http://127.0.0.1" + ":" + pulsar.getListenPortHTTP().get());
        admin = PulsarAdmin.builder().serviceHttpUrl(adminUrl.toString()).build();

        admin.clusters().createCluster("usc", new ClusterData(adminUrl.toString()));
        admin.tenants().createTenant("prop",
                new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet("usc")));
        admin.namespaces().createNamespace("prop/ns-quota");
        admin.namespaces().setNamespaceReplicationClusters("prop/ns-quota", Sets.newHashSet("usc"));
        admin.namespaces().createNamespace("prop/quotahold");
        admin.namespaces().setNamespaceReplicationClusters("prop/quotahold", Sets.newHashSet("usc"));
        admin.namespaces().createNamespace("prop/quotaholdasync");
        admin.namespaces().setNamespaceReplicationClusters("prop/quotaholdasync", Sets.newHashSet("usc"));
    } catch (Throwable t) {
        LOG.error("Error setting up broker test", t);
        fail("Broker test setup failed");
    }
}
 

public void setBkEnsemble(LocalBookkeeperEnsemble bkEnsemble) {
    this.bkEnsemble = bkEnsemble;
}
 

@BeforeMethod
void setup() throws Exception {
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();
    ZkUtils.createFullPathOptimistic(bkEnsemble.getZkClient(),
            SimpleLoadManagerImpl.LOADBALANCER_DYNAMIC_SETTING_STRATEGY_ZPATH,
            "{\"loadBalancerStrategy\":\"leastLoadedServer\"}".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
            CreateMode.PERSISTENT);

    final String localhost = "localhost";
    // start brokers
    for (int i = 0; i < BROKER_COUNT; i++) {


        ServiceConfiguration config = new ServiceConfiguration();
        config.setBrokerServicePort(Optional.ofNullable(brokerNativeBrokerPorts[i]));
        config.setClusterName("use");
        config.setAdvertisedAddress(localhost);
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.of(0));
        config.setBrokerServicePortTls(Optional.of(0));
        config.setWebServicePortTls(Optional.of(0));
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setBrokerServicePort(Optional.of(0));
        config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
        config.setAdvertisedAddress(localhost+i);
        config.setLoadBalancerEnabled(false);

        pulsarServices[i] = new PulsarService(config);
        pulsarServices[i].start();
        brokerWebServicePorts[i] = pulsarServices[i].getListenPortHTTP().get();
        brokerNativeBrokerPorts[i] = pulsarServices[i].getBrokerListenPort().get();

        brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]);
        lookupAddresses[i] = pulsarServices[i].getAdvertisedAddress() + ":" + pulsarServices[i].getListenPortHTTP().get();
        pulsarAdmins[i] = PulsarAdmin.builder().serviceHttpUrl(brokerUrls[i].toString()).build();
    }

    createNamespacePolicies(pulsarServices[0]);

    Thread.sleep(100);
}
 

@BeforeMethod
void setup() throws Exception {
    executor = new ThreadPoolExecutor(1, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>());

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // Start broker 1
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config1.setClusterName("use");
    config1.setWebServicePort(Optional.of(0));
    config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());

    config1.setAdvertisedAddress("localhost");
    config1.setBrokerServicePort(Optional.of(0));
    config1.setBrokerServicePortTls(Optional.of(0));
    config1.setWebServicePortTls(Optional.of(0));
    pulsar1 = new PulsarService(config1);
    pulsar1.start();

    primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get());
    url1 = new URL(pulsar1.getWebServiceAddress());
    admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();

    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config2.setClusterName("use");
    config2.setWebServicePort(Optional.of(0));
    config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config2.setAdvertisedAddress("localhost");
    config2.setBrokerServicePort(Optional.of(0));
    config2.setBrokerServicePortTls(Optional.of(0));
    config2.setWebServicePortTls(Optional.of(0));
    pulsar2 = new PulsarService(config2);
    pulsar2.start();

    secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get());
    url2 = new URL(pulsar2.getWebServiceAddress());
    admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();

    primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
    secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
    nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
    Thread.sleep(100);
}
 

@BeforeMethod
void setup() throws Exception {
    executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>());
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // Start broker 1
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config1.setClusterName("use");
    config1.setWebServicePort(Optional.of(0));
    config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config1.setBrokerServicePort(Optional.of(0));
    config1.setFailureDomainsEnabled(true);
    config1.setLoadBalancerEnabled(true);
    config1.setAdvertisedAddress("localhost");
    createCluster(bkEnsemble.getZkClient(), config1);
    pulsar1 = new PulsarService(config1);
    pulsar1.start();

    primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get());
    url1 = new URL("http://127.0.0.1" + ":" + pulsar1.getListenPortHTTP().get());
    admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();

    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config2.setClusterName("use");
    config2.setWebServicePort(Optional.of(0));
    config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config2.setBrokerServicePort(Optional.of(0));
    config2.setFailureDomainsEnabled(true);
    config2.setAdvertisedAddress("localhost");
    pulsar2 = new PulsarService(config2);
    pulsar2.start();

    secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get());

    url2 = new URL("http://127.0.0.1" + ":" + config2.getWebServicePort().get());
    admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();

    primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
    secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
    nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
    Thread.sleep(100);
}
 

@BeforeClass
void setup() throws Exception {
    log.info("---- Initializing SLAMonitoringTest -----");
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    String[] args = new String[]{
        "--cluster", "my-cluster",
        "--configuration-store", "localhost:" + bkEnsemble.getZookeeperPort(),
        "--initial-num-transaction-coordinators", "16"};

    PulsarTransactionCoordinatorMetadataSetup.main(args);

    // start brokers
    for (int i = 0; i < BROKER_COUNT; i++) {
        ServiceConfiguration config = new ServiceConfiguration();
        config.setBrokerServicePort(Optional.of(0));
        config.setClusterName("my-cluster");
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.of(0));
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setDefaultNumberOfNamespaceBundles(1);
        config.setLoadBalancerEnabled(false);
        configurations[i] = config;

        pulsarServices[i] = new PulsarService(config);
        pulsarServices[i].start();

        pulsarAdmins[i] = PulsarAdmin.builder()
                .serviceHttpUrl(pulsarServices[i].getWebServiceAddress())
                .build();
    }

    Thread.sleep(100);

    PulsarClient client = PulsarClient.builder().
        serviceUrl(pulsarServices[0].getBrokerServiceUrl())
        .build();
    transactionCoordinatorClient = new TransactionCoordinatorClientImpl(client);
    transactionCoordinatorClient.start();

    Thread.sleep(3000);
}
 

@BeforeMethod(timeOut = 300000)
void setup(Method method) throws Exception {
    log.info("--- Setting up method {} ---", method.getName());

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    config = spy(new ServiceConfiguration());
    config.setClusterName("use");
    config.setWebServicePort(Optional.of(0));
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerServicePort(Optional.of(0));
    config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
    config.setTlsAllowInsecureConnection(true);
    config.setAdvertisedAddress("localhost");
    config.setLoadBalancerSheddingEnabled(false);
    config.setLoadBalancerAutoBundleSplitEnabled(false);
    config.setLoadBalancerEnabled(false);
    config.setLoadBalancerAutoUnloadSplitBundlesEnabled(false);

    config.setAllowAutoTopicCreationType("non-partitioned");


    pulsar = new PulsarService(config);
    pulsar.start();

    String brokerServiceUrl = pulsar.getWebServiceAddress();
    url = new URL(brokerServiceUrl);

    admin = PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).build();

    brokerStatsClient = admin.brokerStats();
    primaryHost = pulsar.getWebServiceAddress();

    // update cluster metadata
    ClusterData clusterData = new ClusterData(url.toString());
    admin.clusters().createCluster(config.getClusterName(), clusterData);

    ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).maxBackoffInterval(1, TimeUnit.SECONDS);
    pulsarClient = clientBuilder.build();

    TenantInfo tenantInfo = new TenantInfo();
    tenantInfo.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use")));
    admin.tenants().createTenant(tenant, tenantInfo);
}
 

public void setBkEnsemble(LocalBookkeeperEnsemble bkEnsemble) {
    this.bkEnsemble = bkEnsemble;
}
 

static void start() throws Exception {
  // Start local bookkeeper ensemble
  final LocalBookkeeperEnsemble bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, TestUtil::nextFreePort);
  bkEnsemble.start();

  final String brokerServiceUrl = "http://127.0.0.1:" + brokerWebServicePort;

  final ServiceConfiguration config = new ServiceConfiguration();
  config.setClusterName(CLUSTER_NAME);
  final Set<String> superUsers = Sets.newHashSet("superUser");
  config.setSuperUserRoles(superUsers);

  config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
  config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
  config.setWebServicePort(brokerWebServicePort);
  config.setBrokerServicePort(brokerServicePort);

  config.setAuthenticationEnabled(false);
  config.setTlsEnabled(false);
  config.setTlsAllowInsecureConnection(true);
  config.setAdvertisedAddress("localhost");

  final WorkerService functionsWorkerService = createPulsarFunctionWorker(config);
  final URL urlTls = new URL(brokerServiceUrl);
  final Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService);
  try (final PulsarService pulsar = new PulsarService(config, functionWorkerService)) {
    pulsar.start();
    try (final PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).allowTlsInsecureConnection(true).build()) {
      // update cluster metadata
      final ClusterData clusterData = new ClusterData(urlTls.toString());
      admin.clusters().updateCluster(config.getClusterName(), clusterData);

      final ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(workerConfig.getPulsarServiceUrl());
      try (final PulsarClient pulsarClient = clientBuilder.build()) {
        final TenantInfo propAdmin = new TenantInfo();
        propAdmin.getAdminRoles().add("superUser");
        propAdmin.setAllowedClusters(Sets.newHashSet(CLUSTER_NAME));
        admin.tenants().updateTenant(tenant, propAdmin);

        testPulsarFunction(admin, pulsarClient);
      }
    }
  }
}
 

static void start() throws Exception {
  // Start local bookkeeper ensemble
  final LocalBookkeeperEnsemble bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT,TestUtil::nextFreePort);
  bkEnsemble.start();

  final String brokerServiceUrl = "http://127.0.0.1:" + brokerWebServicePort;

  final ServiceConfiguration config = spy(new ServiceConfiguration());
  config.setClusterName(CLUSTER_NAME);
  final Set<String> superUsers = Sets.newHashSet("superUser");
  config.setSuperUserRoles(superUsers);
  config.setWebServicePort(Optional.of(brokerWebServicePort));
  config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
  config.setBrokerServicePort(Optional.of(brokerServicePort));
  config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
  config.setTlsAllowInsecureConnection(true);
  config.setAdvertisedAddress("localhost");

  config.setAuthenticationEnabled(false);
  config.setAuthorizationEnabled(false);

  config.setBrokerClientTlsEnabled(false);
  config.setAllowAutoTopicCreationType("non-partitioned");

  final WorkerService functionsWorkerService = createPulsarFunctionWorker(config);
  final URL urlTls = new URL(brokerServiceUrl);
  final Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService);
  try (final PulsarService pulsar = new PulsarService(config, functionWorkerService)) {
    pulsar.start();
    try (final PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).allowTlsInsecureConnection(true).build()) {
      // update cluster metadata
      final ClusterData clusterData = new ClusterData(urlTls.toString());
      admin.clusters().updateCluster(config.getClusterName(), clusterData);

      final TenantInfo propAdmin = new TenantInfo();
      propAdmin.getAdminRoles().add("superUser");
      propAdmin.setAllowedClusters(Sets.newHashSet(CLUSTER_NAME));
      admin.tenants().updateTenant(tenant, propAdmin);

      final String jarFilePathUrl = Utils.FILE + ":" + ExclamationFunction.class.getProtectionDomain().getCodeSource().getLocation().getPath();

      final ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(workerConfig.getPulsarServiceUrl());
      try (final PulsarClient pulsarClient = clientBuilder.build()) {
        testE2EPulsarFunction(jarFilePathUrl, admin, pulsarClient);
      }
    }
  }
}
 

public void start() throws Exception {

        if (config == null) {
            throw new IllegalArgumentException("Null configuration is provided");
        }

        if (config.getAdvertisedAddress() != null && !config.getListeners().contains(config.getAdvertisedAddress())) {
            String err = "Error config: advertisedAddress - " + config.getAdvertisedAddress() + " and listeners - "
                + config.getListeners() + " not match.";
            log.error(err);
            throw new IllegalArgumentException(err);
        }

        log.info("--- setup KafkaStandaloneStarter ---");

        if (!this.isOnlyBroker()) {
            ServerConfiguration bkServerConf = new ServerConfiguration();
            bkServerConf.loadConf(new File(configFile).toURI().toURL());

            // Start LocalBookKeeper
            bkEnsemble = new LocalBookkeeperEnsemble(
                    this.getNumOfBk(), this.getZkPort(), this.getBkPort(), this.getStreamStoragePort(), this.getZkDir(),
                    this.getBkDir(), this.isWipeData(), "127.0.0.1");
            bkEnsemble.startStandalone(bkServerConf, !this.isNoStreamStorage());
        }

        if (this.isNoBroker()) {
            return;
        }

        // Start Broker
        kafkaBroker = new KafkaService(config);
        kafkaBroker.start();

        URL webServiceUrl = new URL(
                String.format("http://%s:%d", config.getAdvertisedAddress(), config.getWebServicePort().get()));
        final String brokerServiceUrl = String.format("pulsar://%s:%d", config.getAdvertisedAddress(),
            config.getBrokerServicePort().get());

        admin = PulsarAdmin.builder().serviceHttpUrl(webServiceUrl.toString()).authentication(
                config.getBrokerClientAuthenticationPlugin(), config.getBrokerClientAuthenticationParameters()).build();

        final String cluster = config.getClusterName();

        createDefaultNameSpace(webServiceUrl, brokerServiceUrl, cluster);

        log.info("--- setup completed ---");
    }