Java Code Examples for org.apache.flink.runtime.util.ZooKeeperUtils#createLeaderElectionService()

/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 

/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 

/**
 * Tests the repeated reelection of {@link LeaderContender} once the current leader dies.
 * Furthermore, it tests that new LeaderElectionServices can be started later on and that they
 * successfully register at ZooKeeper and take part in the leader election.
 */
@Test
public void testZooKeeperReelectionWithReplacement() throws Exception {
	int num = 3;
	int numTries = 30;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)" + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		for (int i = 0; i < numTries; i++) {
			listener.waitForNewLeader(timeout);

			String address = listener.getAddress();

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));
				int lastTry = Integer.parseInt(m.group(2));

				assertEquals(listener.getLeaderSessionID(), contenders[index].getLeaderSessionID());

				// stop leader election service = revoke leadership
				leaderElectionService[index].stop();
				// create new leader election service which takes part in the leader election
				leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
				contenders[index] = new TestingContender(
						TEST_URL + "_" + index + "_" + (lastTry + 1),
						leaderElectionService[index]);

				leaderElectionService[index].start(contenders[index]);
			} else {
				throw new Exception("Did not find the leader's index.");
			}
		}

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

@Override
public LeaderElectionService getClusterRestEndpointLeaderElectionService() {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration, REST_SERVER_LEADER_PATH);
}
 

@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration, getPathForJobManager(jobID));
}
 

/**
 * Tests that there is no information left in the ZooKeeper cluster after the ZooKeeper client
 * has terminated. In other words, checks that the ZooKeeperLeaderElection service uses
 * ephemeral nodes.
 */
@Test
public void testEphemeralZooKeeperNodes() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
	TestingContender testingContender;
	TestingListener listener;

	CuratorFramework client = null;
	CuratorFramework client2 = null;
	NodeCache cache = null;

	try {
		client = ZooKeeperUtils.startCuratorFramework(configuration);
		client2 = ZooKeeperUtils.startCuratorFramework(configuration);

		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client2, configuration);
		testingContender = new TestingContender(TEST_URL, leaderElectionService);
		listener = new TestingListener();

		final String leaderPath = configuration.getString(HighAvailabilityOptions.HA_ZOOKEEPER_LEADER_PATH);
		cache = new NodeCache(client2, leaderPath);

		ExistsCacheListener existsListener = new ExistsCacheListener(cache);
		DeletedCacheListener deletedCacheListener = new DeletedCacheListener(cache);

		cache.getListenable().addListener(existsListener);
		cache.start();

		leaderElectionService.start(testingContender);

		testingContender.waitForLeader(timeout);

		Future<Boolean> existsFuture = existsListener.nodeExists();

		existsFuture.get(timeout, TimeUnit.MILLISECONDS);

		cache.getListenable().addListener(deletedCacheListener);

		leaderElectionService.stop();

		// now stop the underlying client
		client.close();

		Future<Boolean> deletedFuture = deletedCacheListener.nodeDeleted();

		// make sure that the leader node has been deleted
		deletedFuture.get(timeout, TimeUnit.MILLISECONDS);

		leaderRetrievalService.start(listener);

		try {
			listener.waitForNewLeader(1000L);

			fail("TimeoutException was expected because there is no leader registered and " +
					"thus there shouldn't be any leader information in ZooKeeper.");
		} catch (TimeoutException e) {
			//that was expected
		}
	} finally {
		if(leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		if (cache != null) {
			cache.close();
		}

		if (client2 != null) {
			client2.close();
		}
	}
}
 

@Override
public LeaderElectionService createLeaderElectionService() throws Exception {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration);
}
 

/**
 * Tests the repeated reelection of {@link LeaderContender} once the current leader dies.
 * Furthermore, it tests that new LeaderElectionServices can be started later on and that they
 * successfully register at ZooKeeper and take part in the leader election.
 */
@Test
public void testZooKeeperReelectionWithReplacement() throws Exception {
	int num = 3;
	int numTries = 30;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)" + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		for (int i = 0; i < numTries; i++) {
			listener.waitForNewLeader(timeout);

			String address = listener.getAddress();

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));
				int lastTry = Integer.parseInt(m.group(2));

				assertEquals(listener.getLeaderSessionID(), contenders[index].getLeaderSessionID());

				// stop leader election service = revoke leadership
				leaderElectionService[index].stop();
				// create new leader election service which takes part in the leader election
				leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
				contenders[index] = new TestingContender(
						TEST_URL + "_" + index + "_" + (lastTry + 1),
						leaderElectionService[index]);

				leaderElectionService[index].start(contenders[index]);
			} else {
				throw new Exception("Did not find the leader's index.");
			}
		}

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(contender.getAddress()) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

@Override
public LeaderElectionService createLeaderElectionService() throws Exception {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration);
}
 

@Override
public LeaderElectionService getWebMonitorLeaderElectionService() {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration, REST_SERVER_LEADER_PATH);
}
 

@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}
 

@Override
public LeaderElectionService getDispatcherLeaderElectionService() {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration, DISPATCHER_LEADER_PATH);
}
 

@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}
 

/**
 * Tests the repeated reelection of {@link LeaderContender} once the current leader dies.
 * Furthermore, it tests that new LeaderElectionServices can be started later on and that they
 * successfully register at ZooKeeper and take part in the leader election.
 */
@Test
public void testZooKeeperReelectionWithReplacement() throws Exception {
	int num = 3;
	int numTries = 30;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)" + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		for (int i = 0; i < numTries; i++) {
			listener.waitForNewLeader(timeout);

			String address = listener.getAddress();

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));
				int lastTry = Integer.parseInt(m.group(2));

				assertEquals(listener.getLeaderSessionID(), contenders[index].getLeaderSessionID());

				// stop leader election service = revoke leadership
				leaderElectionService[index].stop();
				// create new leader election service which takes part in the leader election
				leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
				contenders[index] = new TestingContender(
						TEST_URL + "_" + index + "_" + (lastTry + 1),
						leaderElectionService[index]);

				leaderElectionService[index].start(contenders[index]);
			} else {
				throw new Exception("Did not find the leader's index.");
			}
		}

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(contender.getAddress()) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

@Override
public LeaderElectionService createLeaderElectionService() throws Exception {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration);
}
 

@Override
public LeaderElectionService getWebMonitorLeaderElectionService() {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration, REST_SERVER_LEADER_PATH);
}
 

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(createAddress(i), leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(createAddress(index)) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

@Override
public LeaderElectionService getDispatcherLeaderElectionService() {
	return ZooKeeperUtils.createLeaderElectionService(client, configuration, DISPATCHER_LEADER_PATH);
}