Java Code Examples for org.apache.flink.runtime.util.ZooKeeperUtils#createLeaderRetrievalService()

/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 

/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DISPATCHER_LEADER_PATH);
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}
 

@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, REST_SERVER_LEADER_PATH);
}
 

/**
 *  Test that errors in the {@link LeaderElectionService} are correctly forwarded to the
 *  {@link LeaderContender}.
 */
@Test
public void testExceptionForwarding() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
	TestingListener listener = new TestingListener();
	TestingContender testingContender;

	CuratorFramework client;
	final CreateBuilder mockCreateBuilder = mock(CreateBuilder.class, Mockito.RETURNS_DEEP_STUBS);
	final Exception testException = new Exception("Test exception");

	try {
		client = spy(ZooKeeperUtils.startCuratorFramework(configuration));

		Answer<CreateBuilder> answer = new Answer<CreateBuilder>() {
			private int counter = 0;

			@Override
			public CreateBuilder answer(InvocationOnMock invocation) throws Throwable {
				counter++;

				// at first we have to create the leader latch, there it mustn't fail yet
				if (counter < 2) {
					return (CreateBuilder) invocation.callRealMethod();
				} else {
					return mockCreateBuilder;
				}
			}
		};

		doAnswer(answer).when(client).create();

		when(
			mockCreateBuilder
			.creatingParentsIfNeeded()
			.withMode(Matchers.any(CreateMode.class))
			.forPath(anyString(), any(byte[].class))).thenThrow(testException);

		leaderElectionService = new ZooKeeperLeaderElectionService(client, "/latch", "/leader");
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		testingContender = new TestingContender(TEST_URL, leaderElectionService);

		leaderElectionService.start(testingContender);
		leaderRetrievalService.start(listener);

		testingContender.waitForError(timeout);

		assertNotNull(testingContender.getError());
		assertEquals(testException, testingContender.getError().getCause());
	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 

/**
 * Tests the repeated reelection of {@link LeaderContender} once the current leader dies.
 * Furthermore, it tests that new LeaderElectionServices can be started later on and that they
 * successfully register at ZooKeeper and take part in the leader election.
 */
@Test
public void testZooKeeperReelectionWithReplacement() throws Exception {
	int num = 3;
	int numTries = 30;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)" + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		for (int i = 0; i < numTries; i++) {
			listener.waitForNewLeader(timeout);

			String address = listener.getAddress();

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));
				int lastTry = Integer.parseInt(m.group(2));

				assertEquals(listener.getLeaderSessionID(), contenders[index].getLeaderSessionID());

				// stop leader election service = revoke leadership
				leaderElectionService[index].stop();
				// create new leader election service which takes part in the leader election
				leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
				contenders[index] = new TestingContender(
						TEST_URL + "_" + index + "_" + (lastTry + 1),
						leaderElectionService[index]);

				leaderElectionService[index].start(contenders[index]);
			} else {
				throw new Exception("Did not find the leader's index.");
			}
		}

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(contender.getAddress()) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

@Override
public LeaderRetrievalService getWebMonitorLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, REST_SERVER_LEADER_PATH);
}
 

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(createAddress(i), leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(createAddress(index)) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DISPATCHER_LEADER_PATH);
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}
 

/**
 *  Test that errors in the {@link LeaderElectionService} are correctly forwarded to the
 *  {@link LeaderContender}.
 */
@Test
public void testExceptionForwarding() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
	TestingListener listener = new TestingListener();
	TestingContender testingContender;

	CuratorFramework client;
	final CreateBuilder mockCreateBuilder = mock(CreateBuilder.class, Mockito.RETURNS_DEEP_STUBS);
	final Exception testException = new Exception("Test exception");

	try {
		client = spy(ZooKeeperUtils.startCuratorFramework(configuration));

		Answer<CreateBuilder> answer = new Answer<CreateBuilder>() {
			private int counter = 0;

			@Override
			public CreateBuilder answer(InvocationOnMock invocation) throws Throwable {
				counter++;

				// at first we have to create the leader latch, there it mustn't fail yet
				if (counter < 2) {
					return (CreateBuilder) invocation.callRealMethod();
				} else {
					return mockCreateBuilder;
				}
			}
		};

		doAnswer(answer).when(client).create();

		when(
			mockCreateBuilder
			.creatingParentsIfNeeded()
			.withMode(Matchers.any(CreateMode.class))
			.forPath(anyString(), any(byte[].class))).thenThrow(testException);

		leaderElectionService = new ZooKeeperLeaderElectionService(client, "/latch", "/leader");
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		testingContender = new TestingContender(TEST_URL, leaderElectionService);

		leaderElectionService.start(testingContender);
		leaderRetrievalService.start(listener);

		testingContender.waitForError(timeout);

		assertNotNull(testingContender.getError());
		assertEquals(testException, testingContender.getError().getCause());
	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 

/**
 * Tests the repeated reelection of {@link LeaderContender} once the current leader dies.
 * Furthermore, it tests that new LeaderElectionServices can be started later on and that they
 * successfully register at ZooKeeper and take part in the leader election.
 */
@Test
public void testZooKeeperReelectionWithReplacement() throws Exception {
	int num = 3;
	int numTries = 30;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)" + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		for (int i = 0; i < numTries; i++) {
			listener.waitForNewLeader(timeout);

			String address = listener.getAddress();

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));
				int lastTry = Integer.parseInt(m.group(2));

				assertEquals(listener.getLeaderSessionID(), contenders[index].getLeaderSessionID());

				// stop leader election service = revoke leadership
				leaderElectionService[index].stop();
				// create new leader election service which takes part in the leader election
				leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
				contenders[index] = new TestingContender(
						TEST_URL + "_" + index + "_" + (lastTry + 1),
						leaderElectionService[index]);

				leaderElectionService[index].start(contenders[index]);
			} else {
				throw new Exception("Did not find the leader's index.");
			}
		}

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(contender.getAddress()) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}
 

@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, REST_SERVER_LEADER_PATH);
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DISPATCHER_LEADER_PATH);
}