org.apache.twill.zookeeper.ZKClientService Java Examples
The following examples show how to use
org.apache.twill.zookeeper.ZKClientService.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BalanceBooks.java From phoenix-tephra with Apache License 2.0 | 6 votes |
/** * Sets up common resources required by all clients. */ public void init() throws IOException { Injector injector = Guice.createInjector( new ConfigModule(conf), new ZKModule(), new DiscoveryModules().getDistributedModules(), new TransactionModules().getDistributedModules(), new TransactionClientModule() ); zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); txClient = injector.getInstance(TransactionServiceClient.class); conn = ConnectionFactory.createConnection(conf); createTableIfNotExists(conf, TABLE, new byte[][]{ FAMILY }); }
Example #2
Source File: BalanceBooks.java From phoenix-tephra with Apache License 2.0 | 6 votes |
/** * Sets up common resources required by all clients. */ public void init() throws IOException { Injector injector = Guice.createInjector( new ConfigModule(conf), new ZKModule(), new DiscoveryModules().getDistributedModules(), new TransactionModules().getDistributedModules(), new TransactionClientModule() ); zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); txClient = injector.getInstance(TransactionServiceClient.class); conn = ConnectionFactory.createConnection(conf); createTableIfNotExists(conf, TABLE, new byte[][]{ FAMILY }); }
Example #3
Source File: TephraTransactionProvider.java From phoenix with Apache License 2.0 | 6 votes |
@Override public PhoenixTransactionService getTransactionService(Configuration config, ConnectionInfo connInfo, int port) { config.setInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, port); int retryTimeOut = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_DISCOVERY_TIMEOUT_SEC, TxConstants.Service.DEFAULT_DATA_TX_CLIENT_DISCOVERY_TIMEOUT_SEC); ZKClientService zkClient = ZKClientServices.delegate( ZKClients.reWatchOnExpire( ZKClients.retryOnFailure( ZKClientService.Builder.of(connInfo.getZookeeperConnectionString()) .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT)) .build(), RetryStrategies.exponentialDelay(500, retryTimeOut, TimeUnit.MILLISECONDS) ) ) ); DiscoveryService discovery = new ZKDiscoveryService(zkClient); TransactionManager txManager = new TransactionManager(config, new HDFSTransactionStateStorage(config, new SnapshotCodecProvider(config), new TxMetricsCollector()), new TxMetricsCollector()); TransactionService txService = new TransactionService(config, zkClient, discovery, Providers.of(txManager)); TephraTransactionService service = new TephraTransactionService(zkClient, txService); service.start(); return service; }
Example #4
Source File: BalanceBooks.java From phoenix-tephra with Apache License 2.0 | 6 votes |
/** * Sets up common resources required by all clients. */ public void init() throws IOException { Injector injector = Guice.createInjector( new ConfigModule(conf), new ZKModule(), new DiscoveryModules().getDistributedModules(), new TransactionModules().getDistributedModules(), new TransactionClientModule() ); zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); txClient = injector.getInstance(TransactionServiceClient.class); createTableIfNotExists(conf, TABLE, new byte[][]{ FAMILY }); conn = HConnectionManager.createConnection(conf); }
Example #5
Source File: BalanceBooks.java From phoenix-tephra with Apache License 2.0 | 6 votes |
/** * Sets up common resources required by all clients. */ public void init() throws IOException { Injector injector = Guice.createInjector( new ConfigModule(conf), new ZKModule(), new DiscoveryModules().getDistributedModules(), new TransactionModules().getDistributedModules(), new TransactionClientModule() ); zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); txClient = injector.getInstance(TransactionServiceClient.class); conn = ConnectionFactory.createConnection(conf); createTableIfNotExists(conf, TABLE, new byte[][]{ FAMILY }); }
Example #6
Source File: TransactionServiceMain.java From phoenix-tephra with Apache License 2.0 | 6 votes |
/** * Invoked by jsvc to start the program. */ public void start() throws Exception { Injector injector = Guice.createInjector( new ConfigModule(conf), new ZKModule(), new DiscoveryModules().getDistributedModules(), new TransactionModules().getDistributedModules(), new TransactionClientModule() ); ZKClientService zkClientService = injector.getInstance(ZKClientService.class); zkClientService.startAndWait(); // start a tx server txService = injector.getInstance(TransactionService.class); try { LOG.info("Starting {}", getClass().getSimpleName()); txService.startAndWait(); } catch (Exception e) { System.err.println("Failed to start service: " + e.getMessage()); } }
Example #7
Source File: ZKModule.java From phoenix-tephra with Apache License 2.0 | 6 votes |
@Provides @Singleton private ZKClientService provideZKClientService(Configuration conf) { String zkStr = conf.get(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM); if (zkStr == null) { // Default to HBase one. zkStr = conf.get(TxConstants.HBase.ZOOKEEPER_QUORUM); } int timeOut = conf.getInt(TxConstants.HBase.ZK_SESSION_TIMEOUT, TxConstants.HBase.DEFAULT_ZK_SESSION_TIMEOUT); ZKClientService zkClientService = new TephraZKClientService(zkStr, timeOut, null, ArrayListMultimap.<String, byte[]>create()); return ZKClientServices.delegate( ZKClients.reWatchOnExpire( ZKClients.retryOnFailure(zkClientService, RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS) ) ) ); }
Example #8
Source File: ReentrantDistributedLockTest.java From twill with Apache License 2.0 | 6 votes |
@Test(timeout = 20000) public void testReentrant() { // Test the lock is reentrant from the same thread ZKClientService zkClient = createZKClient(); try { ReentrantDistributedLock lock = new ReentrantDistributedLock(zkClient, "reentrant"); lock.lock(); try { try { lock.lock(); } finally { lock.unlock(); } } finally { lock.unlock(); } } finally { zkClient.stopAndWait(); } }
Example #9
Source File: ThriftTransactionServerTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private void expireZkSession(ZKClientService zkClientService) throws Exception { ZooKeeper zooKeeper = zkClientService.getZooKeeperSupplier().get(); final SettableFuture<?> connectFuture = SettableFuture.create(); Watcher watcher = new Watcher() { @Override public void process(WatchedEvent event) { if (event.getState() == Event.KeeperState.SyncConnected) { connectFuture.set(null); } } }; // Create another Zookeeper session with the same sessionId so that the original one expires. ZooKeeper dupZookeeper = new ZooKeeper(zkClientService.getConnectString(), zooKeeper.getSessionTimeout(), watcher, zooKeeper.getSessionId(), zooKeeper.getSessionPasswd()); connectFuture.get(30, TimeUnit.SECONDS); Assert.assertEquals("Failed to re-create current session", dupZookeeper.getState(), ZooKeeper.States.CONNECTED); dupZookeeper.close(); }
Example #10
Source File: DiscoveryModules.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Provides @Singleton private DiscoveryServiceClient providesDiscoveryServiceClient(final ZKClientService zkClient, final ZKDiscoveryService delegate) { return new DiscoveryServiceClient() { @Override public ServiceDiscovered discover(String s) { if (!zkClient.isRunning()) { zkClient.startAndWait(); } return delegate.discover(s); } }; }
Example #11
Source File: YarnTwillRunnerService.java From twill with Apache License 2.0 | 5 votes |
private ZKClientService getZKClientService(String zkConnect) { return ZKClientServices.delegate( ZKClients.reWatchOnExpire( ZKClients.retryOnFailure(ZKClientService.Builder.of(zkConnect) .setSessionTimeout(ZK_TIMEOUT) .build(), RetryStrategies.exponentialDelay(100, 2000, TimeUnit.MILLISECONDS)))); }
Example #12
Source File: ReentrantDistributedLockTest.java From twill with Apache License 2.0 | 5 votes |
@Test (timeout = 20000) public void testLockInterrupt() throws InterruptedException { // Test lock interruption on multiple threads. ZKClientService zkClient = createZKClient(); try { final ReentrantDistributedLock lock = new ReentrantDistributedLock(zkClient, "/interrupt"); // Create a new thread to acquire the same lock interruptibly. lock.lock(); try { final CountDownLatch lockAcquired = new CountDownLatch(1); final CountDownLatch lockInterrupted = new CountDownLatch(1); Thread t = new Thread() { @Override public void run() { try { lock.lockInterruptibly(); try { lockAcquired.countDown(); } finally { lock.unlock(); } } catch (InterruptedException e) { lockInterrupted.countDown(); } } }; t.start(); t.interrupt(); Assert.assertFalse(lockAcquired.await(2, TimeUnit.SECONDS)); Assert.assertTrue(lockInterrupted.await(2, TimeUnit.SECONDS)); } finally { lock.unlock(); } } finally { zkClient.stopAndWait(); } }
Example #13
Source File: ReentrantDistributedLockTest.java From twill with Apache License 2.0 | 5 votes |
@Test (timeout = 20000) public void testMultiThreads() throws InterruptedException { // Test the lock mechanism between multiple threads ZKClientService zkClient = createZKClient(); try { // Create two threads and compete for the lock final ReentrantDistributedLock lock = new ReentrantDistributedLock(zkClient, "multiThreads"); final CountDownLatch acquired = new CountDownLatch(1); Thread t = new Thread() { @Override public void run() { lock.lock(); try { acquired.countDown(); } finally { lock.unlock(); } } }; lock.lock(); try { t.start(); // Wait for the thread to get the lock, should fail. Assert.assertFalse(acquired.await(1, TimeUnit.SECONDS)); } finally { lock.unlock(); } Assert.assertTrue(acquired.await(5, TimeUnit.SECONDS)); t.join(); } finally { zkClient.stopAndWait(); } }
Example #14
Source File: ZKDiscoveryServiceTest.java From twill with Apache License 2.0 | 5 votes |
@BeforeClass public static void beforeClass() { zkServer = InMemoryZKServer.builder().setTickTime(100000).build(); zkServer.startAndWait(); zkClient = ZKClientServices.delegate( ZKClients.retryOnFailure( ZKClients.reWatchOnExpire( ZKClientService.Builder.of(zkServer.getConnectionStr()).build()), RetryStrategies.fixDelay(1, TimeUnit.SECONDS))); zkClient.startAndWait(); }
Example #15
Source File: KafkaAppender.java From twill with Apache License 2.0 | 5 votes |
@Override public void start() { Preconditions.checkNotNull(zkConnectStr); eventConverter = new LogEventConverter(hostname, runnableName); scheduler = Executors.newSingleThreadScheduledExecutor(Threads.createDaemonThreadFactory(PUBLISH_THREAD_NAME)); zkClientService = ZKClientServices.delegate( ZKClients.reWatchOnExpire( ZKClients.retryOnFailure(ZKClientService.Builder.of(zkConnectStr).build(), RetryStrategies.fixDelay(1, TimeUnit.SECONDS)))); kafkaClient = new ZKKafkaClientService(zkClientService); Futures.addCallback(Services.chainStart(zkClientService, kafkaClient), new FutureCallback<List<ListenableFuture<Service.State>>>() { @Override public void onSuccess(List<ListenableFuture<Service.State>> result) { for (ListenableFuture<Service.State> future : result) { Preconditions.checkState(Futures.getUnchecked(future) == Service.State.RUNNING, "Service is not running."); } addInfo("Kafka client started: " + zkConnectStr); scheduler.scheduleWithFixedDelay(flushTask, 0, flushPeriod, TimeUnit.MILLISECONDS); } @Override public void onFailure(Throwable t) { // Fail to talk to kafka. Other than logging, what can be done? addError("Failed to start kafka appender.", t); } }, Threads.SAME_THREAD_EXECUTOR); super.start(); }
Example #16
Source File: ControllerTest.java From twill with Apache License 2.0 | 5 votes |
@Test public void testController() throws ExecutionException, InterruptedException, TimeoutException { InMemoryZKServer zkServer = InMemoryZKServer.builder().build(); zkServer.startAndWait(); LOG.info("ZKServer: " + zkServer.getConnectionStr()); try { RunId runId = RunIds.generate(); ZKClientService zkClientService = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); zkClientService.startAndWait(); Service service = createService(zkClientService, runId); service.startAndWait(); TwillController controller = getController(zkClientService, "testController", runId); controller.sendCommand(Command.Builder.of("test").build()).get(2, TimeUnit.SECONDS); controller.terminate().get(2, TimeUnit.SECONDS); final CountDownLatch terminateLatch = new CountDownLatch(1); service.addListener(new ServiceListenerAdapter() { @Override public void terminated(Service.State from) { terminateLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(service.state() == Service.State.TERMINATED || terminateLatch.await(2, TimeUnit.SECONDS)); zkClientService.stopAndWait(); } finally { zkServer.stopAndWait(); } }
Example #17
Source File: ControllerTest.java From twill with Apache License 2.0 | 5 votes |
@Test public void testControllerBefore() throws InterruptedException, ExecutionException, TimeoutException { InMemoryZKServer zkServer = InMemoryZKServer.builder().build(); zkServer.startAndWait(); LOG.info("ZKServer: " + zkServer.getConnectionStr()); try { RunId runId = RunIds.generate(); ZKClientService zkClientService = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); zkClientService.startAndWait(); final CountDownLatch runLatch = new CountDownLatch(1); TwillController controller = getController(zkClientService, "testControllerBefore", runId); controller.onRunning(new Runnable() { @Override public void run() { runLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Service service = createService(zkClientService, runId); service.start(); Assert.assertTrue(runLatch.await(2, TimeUnit.SECONDS)); try { controller.awaitTerminated(2, TimeUnit.SECONDS); Assert.fail("Service should not be terminated"); } catch (TimeoutException e) { // Expected } service.stop(); controller.awaitTerminated(120, TimeUnit.SECONDS); } finally { zkServer.stopAndWait(); } }
Example #18
Source File: ControllerTest.java From twill with Apache License 2.0 | 5 votes |
@Test public void testControllerListener() throws InterruptedException { InMemoryZKServer zkServer = InMemoryZKServer.builder().build(); zkServer.startAndWait(); LOG.info("ZKServer: " + zkServer.getConnectionStr()); try { RunId runId = RunIds.generate(); ZKClientService zkClientService = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); zkClientService.startAndWait(); Service service = createService(zkClientService, runId); service.startAndWait(); final CountDownLatch runLatch = new CountDownLatch(1); TwillController controller = getController(zkClientService, "testControllerListener", runId); controller.onRunning(new Runnable() { @Override public void run() { runLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(runLatch.await(2, TimeUnit.SECONDS)); service.stopAndWait(); zkClientService.stopAndWait(); } finally { zkServer.stopAndWait(); } }
Example #19
Source File: KafkaTest.java From twill with Apache License 2.0 | 5 votes |
@BeforeClass public static void init() throws Exception { zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build(); zkServer.startAndWait(); // Extract the kafka.tgz and start the kafka server kafkaServer = new EmbeddedKafkaServer(generateKafkaConfig(zkServer.getConnectionStr())); kafkaServer.startAndWait(); zkClientService = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); kafkaClient = new ZKKafkaClientService(zkClientService); Services.chainStart(zkClientService, kafkaClient).get(); }
Example #20
Source File: DiscoveryModules.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Provides @Singleton private DiscoveryService providesDiscoveryService(final ZKClientService zkClient, final ZKDiscoveryService delegate) { return new DiscoveryService() { @Override public Cancellable register(Discoverable discoverable) { if (!zkClient.isRunning()) { zkClient.startAndWait(); } return delegate.register(discoverable); } }; }
Example #21
Source File: ApplicationMasterMain.java From twill with Apache License 2.0 | 5 votes |
private void doMain() throws Exception { RunId runId = twillRuntimeSpec.getTwillAppRunId(); ZKClientService zkClientService = createZKClient(); Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration())); setRMSchedulerAddress(conf, twillRuntimeSpec.getRmSchedulerAddr()); final YarnAMClient amClient = new VersionDetectYarnAMClientFactory(conf).create(); ApplicationMasterService service = new ApplicationMasterService(runId, zkClientService, twillRuntimeSpec, amClient, conf, createAppLocation(conf, twillRuntimeSpec.getFsUser(), twillRuntimeSpec.getTwillAppDir())); TrackerService trackerService = new TrackerService(service); List<Service> prerequisites = Lists.newArrayList( new YarnAMClientService(amClient, trackerService), zkClientService, new AppMasterTwillZKPathService(zkClientService, runId) ); if (twillRuntimeSpec.isLogCollectionEnabled()) { prerequisites.add(new ApplicationKafkaService(zkClientService, twillRuntimeSpec.getKafkaZKConnect())); } else { LOG.info("Log collection through kafka disabled"); } new ApplicationMasterMain(twillRuntimeSpec) .doMain( service, prerequisites.toArray(new Service[prerequisites.size()]) ); }
Example #22
Source File: ZKModule.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override protected void configure() { /** * ZKClientService is provided by the provider method * {@link #provideZKClientService(org.apache.hadoop.conf.Configuration)}. */ bind(ZKClient.class).to(ZKClientService.class); }
Example #23
Source File: ServiceMain.java From twill with Apache License 2.0 | 5 votes |
/** * Creates a {@link ZKClientService}. */ protected final ZKClientService createZKClient() { TwillRuntimeSpecification twillRuntimeSpec = getTwillRuntimeSpecification(); return ZKClientServices.delegate( ZKClients.namespace( ZKClients.reWatchOnExpire( ZKClients.retryOnFailure( ZKClientService.Builder.of(twillRuntimeSpec.getZkConnectStr()).build(), RetryStrategies.fixDelay(1, TimeUnit.SECONDS) ) ), "/" + twillRuntimeSpec.getTwillAppName() ) ); }
Example #24
Source File: LeaderElectionTest.java From twill with Apache License 2.0 | 4 votes |
@Test(timeout = 150000) public void testDisconnect() throws IOException, InterruptedException { File zkDataDir = tmpFolder.newFolder(); InMemoryZKServer ownZKServer = InMemoryZKServer.builder().setDataDir(zkDataDir).build(); ownZKServer.startAndWait(); try { ZKClientService zkClient = ZKClientService.Builder.of(ownZKServer.getConnectionStr()).build(); zkClient.startAndWait(); try { final Semaphore leaderSem = new Semaphore(0); final Semaphore followerSem = new Semaphore(0); LeaderElection leaderElection = new LeaderElection(zkClient, "/testDisconnect", new ElectionHandler() { @Override public void leader() { leaderSem.release(); } @Override public void follower() { followerSem.release(); } }); leaderElection.start(); leaderSem.tryAcquire(20, TimeUnit.SECONDS); int zkPort = ownZKServer.getLocalAddress().getPort(); // Disconnect by shutting the server and restart it on the same port ownZKServer.stopAndWait(); // Right after disconnect, it should become follower followerSem.tryAcquire(20, TimeUnit.SECONDS); ownZKServer = InMemoryZKServer.builder().setDataDir(zkDataDir).setPort(zkPort).build(); ownZKServer.startAndWait(); // Right after reconnect, it should be leader again. leaderSem.tryAcquire(20, TimeUnit.SECONDS); // Now disconnect it again, but then cancel it before reconnect, it shouldn't become leader ownZKServer.stopAndWait(); // Right after disconnect, it should become follower followerSem.tryAcquire(20, TimeUnit.SECONDS); ListenableFuture<?> cancelFuture = leaderElection.stop(); ownZKServer = InMemoryZKServer.builder().setDataDir(zkDataDir).setPort(zkPort).build(); ownZKServer.startAndWait(); Futures.getUnchecked(cancelFuture); // After reconnect, it should not be leader Assert.assertFalse(leaderSem.tryAcquire(10, TimeUnit.SECONDS)); } finally { zkClient.stopAndWait(); } } finally { ownZKServer.stopAndWait(); } }
Example #25
Source File: TwillContainerMain.java From twill with Apache License 2.0 | 4 votes |
private void doMain() throws Exception { // Try to load the secure store from localized file, which AM requested RM to localize it for this container. loadSecureStore(); RunId appRunId = twillRuntimeSpec.getTwillAppRunId(); RunId runId = RunIds.fromString(System.getenv(EnvKeys.TWILL_RUN_ID)); String runnableName = System.getenv(EnvKeys.TWILL_RUNNABLE_NAME); int instanceId = Integer.valueOf(System.getenv(EnvKeys.TWILL_INSTANCE_ID)); int instanceCount = Integer.valueOf(System.getenv(EnvKeys.TWILL_INSTANCE_COUNT)); Map<String, String> defaultLogLevels = twillRuntimeSpec.getLogLevels().get(runnableName); Map<String, String> dynamicLogLevels = loadLogLevels().get(runnableName); Map<String, String> logLevels = new HashMap<>(); if (defaultLogLevels != null) { logLevels.putAll(defaultLogLevels); } else { defaultLogLevels = ImmutableMap.of(); } if (dynamicLogLevels != null) { logLevels.putAll(dynamicLogLevels); } ZKClientService zkClientService = createZKClient(); ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClientService); ZKClient appRunZkClient = getAppRunZKClient(zkClientService, appRunId); TwillRunnableSpecification runnableSpec = twillRuntimeSpec.getTwillSpecification().getRunnables().get(runnableName).getRunnableSpecification(); ContainerInfo containerInfo = new EnvContainerInfo(); Arguments arguments = decodeArgs(); BasicTwillContext context = new BasicTwillContext( runId, appRunId, containerInfo.getHost(), arguments.getRunnableArguments().get(runnableName).toArray(new String[0]), arguments.getArguments().toArray(new String[0]), runnableSpec, instanceId, discoveryService, discoveryService, appRunZkClient, instanceCount, containerInfo.getMemoryMB(), containerInfo.getVirtualCores() ); ZKClient containerZKClient = getContainerZKClient(zkClientService, appRunId, runnableName); Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration())); TwillContainerService service = new TwillContainerService(context, containerInfo, containerZKClient, runId, runnableSpec, getClassLoader(), conf, createAppLocation(conf, twillRuntimeSpec.getFsUser(), twillRuntimeSpec.getTwillAppDir()), defaultLogLevels, logLevels); doMain( service, zkClientService, new LogFlushService(), new TwillZKPathService(containerZKClient, runId), new CloseableServiceWrapper(discoveryService) ); }
Example #26
Source File: LeaderElectionTest.java From twill with Apache License 2.0 | 4 votes |
@Test public void testRace() throws InterruptedException { ExecutorService executor = Executors.newFixedThreadPool(2); final AtomicInteger leaderCount = new AtomicInteger(0); final CountDownLatch completeLatch = new CountDownLatch(2); // Starts two threads and try to compete for leader and immediate drop leadership. // This is to test the case when a follower tries to watch for leader node, but the leader is already gone for (int i = 0; i < 2; i++) { final ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); zkClient.startAndWait(); executor.execute(new Runnable() { @Override public void run() { try { for (int i = 0; i < 1000; i++) { final CountDownLatch leaderLatch = new CountDownLatch(1); LeaderElection election = new LeaderElection(zkClient, "/testRace", new ElectionHandler() { @Override public void leader() { leaderCount.incrementAndGet(); leaderLatch.countDown(); } @Override public void follower() { // no-op } }); election.startAndWait(); Uninterruptibles.awaitUninterruptibly(leaderLatch); election.stopAndWait(); } completeLatch.countDown(); } finally { zkClient.stopAndWait(); } } }); } try { Assert.assertTrue(completeLatch.await(2, TimeUnit.MINUTES)); } finally { executor.shutdownNow(); } }
Example #27
Source File: ReentrantDistributedLockTest.java From twill with Apache License 2.0 | 4 votes |
@Test (timeout = 20000) public void testReentrantMultiClients() throws InterruptedException { // Test the reentrant lock/unlock mechanism works correctly across multiple instances of lock // This is to simulate the lock between multiple processes ZKClientService zkClient1 = createZKClient(); ZKClientService zkClient2 = createZKClient(); try { // Create two distributed locks ReentrantDistributedLock lock1 = new ReentrantDistributedLock(zkClient1, "multiClients"); final ReentrantDistributedLock lock2 = new ReentrantDistributedLock(zkClient2, "multiClients"); // Create a thread for locking with lock2 final CountDownLatch lockAcquired = new CountDownLatch(1); Thread t = new Thread() { @Override public void run() { lock2.lock(); try { lockAcquired.countDown(); } finally { lock2.unlock(); } } }; // Lock with lock1 lock1.lock(); try { // Start the lock2 thread. It should be blocked at acquiring the lock t.start(); Assert.assertFalse(lockAcquired.await(1, TimeUnit.SECONDS)); // Lock with lock1 again and unlock. The inner unlock shouldn't release the distributed lock lock1.lock(); lock1.unlock(); Assert.assertFalse(lockAcquired.await(1, TimeUnit.SECONDS)); } finally { lock1.unlock(); } // Now lock1 is unlocked, the lock2 thread should be able to proceed Assert.assertTrue(lockAcquired.await(5, TimeUnit.SECONDS)); t.join(); } finally { zkClient1.stopAndWait(); zkClient2.stopAndWait(); } }
Example #28
Source File: EchoServerTestRun.java From twill with Apache License 2.0 | 4 votes |
@Test public void testZKCleanup() throws Exception { final ZKClientService zkClient = ZKClientService.Builder.of(getZKConnectionString() + "/twill").build(); zkClient.startAndWait(); try { TwillRunner runner = getTwillRunner(); // Start an application and stop it. TwillController controller = runner.prepare(new EchoServer()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationArguments("echo") .withArguments("EchoServer", "echo2") .start(); Iterable<Discoverable> echoServices = controller.discoverService("echo"); Assert.assertTrue(waitForSize(echoServices, 1, 120)); controller.terminate().get(); // Verify the ZK node gets cleanup waitFor(null, new Callable<Stat>() { @Override public Stat call() throws Exception { return zkClient.exists("/EchoServer").get(); } }, 10000, 100, TimeUnit.MILLISECONDS); // Start two instances of the application and stop one of it List<TwillController> controllers = new ArrayList<>(); for (int i = 0; i < 2; i++) { controllers.add(runner.prepare(new EchoServer()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationArguments("echo") .withArguments("EchoServer", "echo2") .start()); } // There should be two instances up and running. echoServices = controllers.get(1).discoverService("echo"); Assert.assertTrue(waitForSize(echoServices, 2, 120)); // Stop one instance of the app controllers.get(0).terminate().get(); // Verify the ZK node should still be there Assert.assertNotNull(zkClient.exists("/EchoServer").get()); // We should still be able to do discovery, which depends on the ZK node. echoServices = controllers.get(1).discoverService("echo"); Assert.assertTrue(waitForSize(echoServices, 1, 120)); // Stop second instance of the app controllers.get(1).terminate().get(); // Verify the ZK node gets cleanup waitFor(null, new Callable<Stat>() { @Override public Stat call() throws Exception { return zkClient.exists("/EchoServer").get(); } }, 10000, 100, TimeUnit.MILLISECONDS); } finally { zkClient.stopAndWait(); } }
Example #29
Source File: KafkaTest.java From twill with Apache License 2.0 | 4 votes |
@Test public void testKafkaClientReconnect() throws Exception { String topic = "backoff"; Properties kafkaServerConfig = generateKafkaConfig(zkServer.getConnectionStr() + "/backoff"); EmbeddedKafkaServer server = new EmbeddedKafkaServer(kafkaServerConfig); ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr() + "/backoff").build(); zkClient.startAndWait(); try { zkClient.create("/", null, CreateMode.PERSISTENT).get(); ZKKafkaClientService kafkaClient = new ZKKafkaClientService(zkClient); kafkaClient.startAndWait(); try { server.startAndWait(); try { // Publish a messages createPublishThread(kafkaClient, topic, Compression.NONE, "First message", 1).start(); // Create a consumer final BlockingQueue<String> queue = new LinkedBlockingQueue<>(); Cancellable cancel = kafkaClient.getConsumer().prepare().add(topic, 0, 0) .consume(new KafkaConsumer.MessageCallback() { @Override public long onReceived(Iterator<FetchedMessage> messages) { long nextOffset = -1L; while (messages.hasNext()) { FetchedMessage message = messages.next(); nextOffset = message.getNextOffset(); queue.offer(Charsets.UTF_8.decode(message.getPayload()).toString()); } return nextOffset; } @Override public void finished() { } }); // Wait for the first message Assert.assertEquals("0 First message", queue.poll(60, TimeUnit.SECONDS)); // Shutdown the server server.stopAndWait(); // Start the server again. // Needs to create a new instance with the same config since guava service cannot be restarted server = new EmbeddedKafkaServer(kafkaServerConfig); server.startAndWait(); // Wait a little while to make sure changes is reflected in broker service TimeUnit.SECONDS.sleep(3); // Publish another message createPublishThread(kafkaClient, topic, Compression.NONE, "Second message", 1).start(); // Should be able to get the second message Assert.assertEquals("0 Second message", queue.poll(60, TimeUnit.SECONDS)); cancel.cancel(); } finally { kafkaClient.stopAndWait(); } } finally { server.stopAndWait(); } } finally { zkClient.stopAndWait(); } }
Example #30
Source File: KafkaTest.java From twill with Apache License 2.0 | 4 votes |
@Test public void testBrokerChange() throws Exception { // Create a new namespace in ZK for Kafka server for this test case String connectionStr = zkServer.getConnectionStr() + "/broker_change"; ZKClientService zkClient = ZKClientService.Builder.of(connectionStr).build(); zkClient.startAndWait(); zkClient.create("/", null, CreateMode.PERSISTENT).get(); // Start a new kafka server File logDir = TMP_FOLDER.newFolder(); EmbeddedKafkaServer server = new EmbeddedKafkaServer(generateKafkaConfig(connectionStr, logDir)); server.startAndWait(); // Start a Kafka client KafkaClientService kafkaClient = new ZKKafkaClientService(zkClient); kafkaClient.startAndWait(); // Attach a consumer final BlockingQueue<String> consumedMessages = Queues.newLinkedBlockingQueue(); kafkaClient.getConsumer() .prepare().addFromBeginning("test", 0).consume(new KafkaConsumer.MessageCallback() { @Override public long onReceived(Iterator<FetchedMessage> messages) { long nextOffset = -1L; while (messages.hasNext()) { FetchedMessage message = messages.next(); nextOffset = message.getNextOffset(); consumedMessages.add(Charsets.UTF_8.decode(message.getPayload()).toString()); } return nextOffset; } @Override public void finished() { // No-op } }); // Get a publisher and publish a message KafkaPublisher publisher = kafkaClient.getPublisher(KafkaPublisher.Ack.FIRE_AND_FORGET, Compression.NONE); publisher.prepare("test").add(Charsets.UTF_8.encode("Message 0"), 0).send().get(); // Should receive one message Assert.assertEquals("Message 0", consumedMessages.poll(5, TimeUnit.SECONDS)); // Now shutdown and restart the server on different port server.stopAndWait(); server = new EmbeddedKafkaServer(generateKafkaConfig(connectionStr, logDir)); server.startAndWait(); // Wait a little while to make sure changes is reflected in broker service TimeUnit.SECONDS.sleep(3); // Now publish again with the same publisher. It should succeed and the consumer should receive the message. publisher.prepare("test").add(Charsets.UTF_8.encode("Message 1"), 0).send().get(); Assert.assertEquals("Message 1", consumedMessages.poll(5, TimeUnit.SECONDS)); kafkaClient.stopAndWait(); zkClient.stopAndWait(); server.stopAndWait(); }