org.apache.flink.queryablestate.network.stats.DisabledKvStateRequestStats Java Examples

/**
 * Tests that in case of port collision, a FlinkRuntimeException is thrown
 * with a specific message.
 */
@Test
public void testServerInitializationFailure() throws Throwable {

	// the expected exception along with the adequate message
	expectedEx.expect(FlinkRuntimeException.class);
	expectedEx.expectMessage("Unable to start Test Server 2. All ports in provided range are occupied.");

	List<Integer> portList = new ArrayList<>();
	portList.add(7777);

	try (TestServer server1 = new TestServer("Test Server 1", new DisabledKvStateRequestStats(), portList.iterator())) {
		server1.start();

		try (TestServer server2 = new TestServer("Test Server 2", new DisabledKvStateRequestStats(),
				Collections.singletonList(server1.getServerAddress().getPort()).iterator())) {
			server2.start();
		}
	}
}
 

/**
 * Create the Queryable State Client.
 * @param remoteAddress the {@link InetAddress address} of the {@code Client Proxy} to connect to.
 * @param remotePort the port of the proxy to connect to.
 */
public QueryableStateClient(final InetAddress remoteAddress, final int remotePort) {
	Preconditions.checkArgument(NetUtils.isValidHostPort(remotePort),
			"Remote Port " + remotePort + " is out of valid port range [0-65535].");

	this.remoteAddress = new InetSocketAddress(remoteAddress, remotePort);

	final MessageSerializer<KvStateRequest, KvStateResponse> messageSerializer =
			new MessageSerializer<>(
					new KvStateRequest.KvStateRequestDeserializer(),
					new KvStateResponse.KvStateResponseDeserializer());

	this.client = new Client<>(
			"Queryable State Client",
			1,
			messageSerializer,
			new DisabledKvStateRequestStats());
}
 

/**
 * Tests that in case of port collision, a FlinkRuntimeException is thrown
 * with a specific message.
 */
@Test
public void testServerInitializationFailure() throws Throwable {

	// the expected exception along with the adequate message
	expectedEx.expect(FlinkRuntimeException.class);
	expectedEx.expectMessage("Unable to start Test Server 2. All ports in provided range are occupied.");

	List<Integer> portList = new ArrayList<>();
	portList.add(7777);

	try (TestServer server1 = new TestServer("Test Server 1", new DisabledKvStateRequestStats(), portList.iterator())) {
		server1.start();

		try (TestServer server2 = new TestServer("Test Server 2", new DisabledKvStateRequestStats(),
				Collections.singletonList(server1.getServerAddress().getPort()).iterator())) {
			server2.start();
		}
	}
}
 

/**
 * Create the Queryable State Client.
 * @param remoteAddress the {@link InetAddress address} of the {@code Client Proxy} to connect to.
 * @param remotePort the port of the proxy to connect to.
 */
public QueryableStateClient(final InetAddress remoteAddress, final int remotePort) {
	Preconditions.checkArgument(remotePort >= 0 && remotePort <= 65536,
			"Remote Port " + remotePort + " is out of valid port range (0-65536).");

	this.remoteAddress = new InetSocketAddress(remoteAddress, remotePort);

	final MessageSerializer<KvStateRequest, KvStateResponse> messageSerializer =
			new MessageSerializer<>(
					new KvStateRequest.KvStateRequestDeserializer(),
					new KvStateResponse.KvStateResponseDeserializer());

	this.client = new Client<>(
			"Queryable State Client",
			1,
			messageSerializer,
			new DisabledKvStateRequestStats());
}
 

/**
 * Tests that in case of port collision, a FlinkRuntimeException is thrown
 * with a specific message.
 */
@Test
public void testServerInitializationFailure() throws Throwable {

	// the expected exception along with the adequate message
	expectedEx.expect(FlinkRuntimeException.class);
	expectedEx.expectMessage("Unable to start Test Server 2. All ports in provided range are occupied.");

	List<Integer> portList = new ArrayList<>();
	portList.add(7777);

	try (TestServer server1 = new TestServer("Test Server 1", new DisabledKvStateRequestStats(), portList.iterator())) {
		server1.start();

		try (TestServer server2 = new TestServer("Test Server 2", new DisabledKvStateRequestStats(),
				Collections.singletonList(server1.getServerAddress().getPort()).iterator())) {
			server2.start();
		}
	}
}
 

/**
 * Create the Queryable State Client.
 * @param remoteAddress the {@link InetAddress address} of the {@code Client Proxy} to connect to.
 * @param remotePort the port of the proxy to connect to.
 */
public QueryableStateClient(final InetAddress remoteAddress, final int remotePort) {
	Preconditions.checkArgument(remotePort >= 0 && remotePort <= 65536,
			"Remote Port " + remotePort + " is out of valid port range (0-65536).");

	this.remoteAddress = new InetSocketAddress(remoteAddress, remotePort);

	final MessageSerializer<KvStateRequest, KvStateResponse> messageSerializer =
			new MessageSerializer<>(
					new KvStateRequest.KvStateRequestDeserializer(),
					new KvStateResponse.KvStateResponseDeserializer());

	this.client = new Client<>(
			"Queryable State Client",
			1,
			messageSerializer,
			new DisabledKvStateRequestStats());
}
 

/**
 * Creates and returns the KvState service.
 *
 * @param taskManagerServicesConfiguration task manager configuration
 * @return service for kvState related components
 */
public static KvStateService fromConfiguration(TaskManagerServicesConfiguration taskManagerServicesConfiguration) {
	KvStateRegistry kvStateRegistry = new KvStateRegistry();

	QueryableStateConfiguration qsConfig = taskManagerServicesConfiguration.getQueryableStateConfig();

	KvStateClientProxy kvClientProxy = null;
	KvStateServer kvStateServer = null;

	if (qsConfig != null) {
		int numProxyServerNetworkThreads = qsConfig.numProxyServerThreads() == 0 ?
			taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numProxyServerThreads();
		int numProxyServerQueryThreads = qsConfig.numProxyQueryThreads() == 0 ?
			taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numProxyQueryThreads();
		kvClientProxy = QueryableStateUtils.createKvStateClientProxy(
			taskManagerServicesConfiguration.getTaskManagerAddress(),
			qsConfig.getProxyPortRange(),
			numProxyServerNetworkThreads,
			numProxyServerQueryThreads,
			new DisabledKvStateRequestStats());

		int numStateServerNetworkThreads = qsConfig.numStateServerThreads() == 0 ?
			taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numStateServerThreads();
		int numStateServerQueryThreads = qsConfig.numStateQueryThreads() == 0 ?
			taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numStateQueryThreads();
		kvStateServer = QueryableStateUtils.createKvStateServer(
			taskManagerServicesConfiguration.getTaskManagerAddress(),
			qsConfig.getStateServerPortRange(),
			numStateServerNetworkThreads,
			numStateServerQueryThreads,
			kvStateRegistry,
			new DisabledKvStateRequestStats());
	}

	return new KvStateService(kvStateRegistry, kvStateServer, kvClientProxy);
}
 

private static Client<KvStateInternalRequest, KvStateResponse> createInternalClient(int threads) {
	final MessageSerializer<KvStateInternalRequest, KvStateResponse> messageSerializer =
			new MessageSerializer<>(
					new KvStateInternalRequest.KvStateInternalRequestDeserializer(),
					new KvStateResponse.KvStateResponseDeserializer());

	return new Client<>(
			"Queryable State Proxy Client",
			threads,
			messageSerializer,
			new DisabledKvStateRequestStats());
}
 

@Before
public void setup() {
	kvStateClientProxy = new KvStateClientProxyImpl(
		InetAddress.getLoopbackAddress(),
		Collections.singleton(0).iterator(),
		1,
		1,
		new DisabledKvStateRequestStats());
}
 

@BeforeClass
public static void setup() {
	try {
		testServer = new KvStateServerImpl(
				InetAddress.getLocalHost(),
				Collections.singletonList(0).iterator(),
				1,
				1,
				new KvStateRegistry(),
				new DisabledKvStateRequestStats());
		testServer.start();
	} catch (Throwable e) {
		e.printStackTrace();
	}
}
 

@Before
public void setup() {
	kvStateClientProxy = new KvStateClientProxyImpl(
		InetAddress.getLoopbackAddress(),
		Collections.singleton(0).iterator(),
		1,
		1,
		new DisabledKvStateRequestStats());
}
 

/**
 * Creates and returns the KvState service.
 *
 * @param taskManagerServicesConfiguration task manager configuration
 * @return service for kvState related components
 */
public static KvStateService fromConfiguration(TaskManagerServicesConfiguration taskManagerServicesConfiguration) {
	KvStateRegistry kvStateRegistry = new KvStateRegistry();

	QueryableStateConfiguration qsConfig = taskManagerServicesConfiguration.getQueryableStateConfig();

	KvStateClientProxy kvClientProxy = null;
	KvStateServer kvStateServer = null;

	if (qsConfig != null) {
		int numProxyServerNetworkThreads = qsConfig.numProxyServerThreads() == 0 ?
			taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numProxyServerThreads();
		int numProxyServerQueryThreads = qsConfig.numProxyQueryThreads() == 0 ?
			taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numProxyQueryThreads();
		kvClientProxy = QueryableStateUtils.createKvStateClientProxy(
			taskManagerServicesConfiguration.getExternalAddress(),
			qsConfig.getProxyPortRange(),
			numProxyServerNetworkThreads,
			numProxyServerQueryThreads,
			new DisabledKvStateRequestStats());

		int numStateServerNetworkThreads = qsConfig.numStateServerThreads() == 0 ?
			taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numStateServerThreads();
		int numStateServerQueryThreads = qsConfig.numStateQueryThreads() == 0 ?
			taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numStateQueryThreads();
		kvStateServer = QueryableStateUtils.createKvStateServer(
			taskManagerServicesConfiguration.getExternalAddress(),
			qsConfig.getStateServerPortRange(),
			numStateServerNetworkThreads,
			numStateServerQueryThreads,
			kvStateRegistry,
			new DisabledKvStateRequestStats());
	}

	return new KvStateService(kvStateRegistry, kvStateServer, kvClientProxy);
}
 

private static Client<KvStateInternalRequest, KvStateResponse> createInternalClient(int threads) {
	final MessageSerializer<KvStateInternalRequest, KvStateResponse> messageSerializer =
			new MessageSerializer<>(
					new KvStateInternalRequest.KvStateInternalRequestDeserializer(),
					new KvStateResponse.KvStateResponseDeserializer());

	return new Client<>(
			"Queryable State Proxy Client",
			threads,
			messageSerializer,
			new DisabledKvStateRequestStats());
}
 

@BeforeClass
public static void setup() {
	try {
		testServer = new KvStateServerImpl(
				InetAddress.getLocalHost(),
				Collections.singletonList(0).iterator(),
				1,
				1,
				new KvStateRegistry(),
				new DisabledKvStateRequestStats());
		testServer.start();
	} catch (Throwable e) {
		e.printStackTrace();
	}
}
 

@BeforeClass
public static void setup() {
	try {
		testServer = new KvStateServerImpl(
				InetAddress.getLocalHost().getHostName(),
				Collections.singletonList(0).iterator(),
				1,
				1,
				new KvStateRegistry(),
				new DisabledKvStateRequestStats());
		testServer.start();
	} catch (Throwable e) {
		e.printStackTrace();
	}
}
 

@Before
public void setup() {
	kvStateClientProxy = new KvStateClientProxyImpl(
		InetAddress.getLoopbackAddress().getHostName(),
		Collections.singleton(0).iterator(),
		1,
		1,
		new DisabledKvStateRequestStats());
}
 

private static Client<KvStateInternalRequest, KvStateResponse> createInternalClient(int threads) {
	final MessageSerializer<KvStateInternalRequest, KvStateResponse> messageSerializer =
			new MessageSerializer<>(
					new KvStateInternalRequest.KvStateInternalRequestDeserializer(),
					new KvStateResponse.KvStateResponseDeserializer());

	return new Client<>(
			"Queryable State Proxy Client",
			threads,
			messageSerializer,
			new DisabledKvStateRequestStats());
}
 

/**
 * Tests the failure response on a rejected execution, because the query executor has been closed.
 */
@Test
public void testQueryExecutorShutDown() throws Throwable {
	KvStateRegistry registry = new KvStateRegistry();
	AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();

	KvStateServerImpl localTestServer = new KvStateServerImpl(
			InetAddress.getLocalHost(),
			Collections.singletonList(0).iterator(),
			1,
			1,
			new KvStateRegistry(),
			new DisabledKvStateRequestStats());

	localTestServer.start();
	localTestServer.shutdown();
	assertTrue(localTestServer.getQueryExecutor().isTerminated());

	MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer =
			new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());

	KvStateServerHandler handler = new KvStateServerHandler(localTestServer, registry, serializer, stats);
	EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);

	int numKeyGroups = 1;
	AbstractStateBackend abstractBackend = new MemoryStateBackend();
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	dummyEnv.setKvStateRegistry(registry);
	KeyedStateBackend<Integer> backend = createKeyedStateBackend(registry, numKeyGroups, abstractBackend, dummyEnv);

	final TestRegistryListener registryListener = new TestRegistryListener();
	registry.registerListener(dummyEnv.getJobID(), registryListener);

	// Register state
	ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
	desc.setQueryable("vanilla");

	backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);

	assertTrue(registryListener.registrationName.equals("vanilla"));

	KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, new byte[0]);
	ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);

	// Write the request and wait for the response
	channel.writeInbound(serRequest);

	ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
	buf.skipBytes(4); // skip frame length

	// Verify the response
	assertEquals(MessageType.REQUEST_FAILURE, MessageSerializer.deserializeHeader(buf));
	RequestFailure response = MessageSerializer.deserializeRequestFailure(buf);

	assertTrue(response.getCause().getMessage().contains("RejectedExecutionException"));

	assertEquals(1L, stats.getNumRequests());
	assertEquals(1L, stats.getNumFailed());

	localTestServer.shutdown();
}
 

/**
 * Tests the failure response on a rejected execution, because the query executor has been closed.
 */
@Test
public void testQueryExecutorShutDown() throws Throwable {
	KvStateRegistry registry = new KvStateRegistry();
	AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();

	KvStateServerImpl localTestServer = new KvStateServerImpl(
			InetAddress.getLocalHost(),
			Collections.singletonList(0).iterator(),
			1,
			1,
			new KvStateRegistry(),
			new DisabledKvStateRequestStats());

	localTestServer.start();
	localTestServer.shutdown();
	assertTrue(localTestServer.getQueryExecutor().isTerminated());

	MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer =
			new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());

	KvStateServerHandler handler = new KvStateServerHandler(localTestServer, registry, serializer, stats);
	EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);

	int numKeyGroups = 1;
	AbstractStateBackend abstractBackend = new MemoryStateBackend();
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	dummyEnv.setKvStateRegistry(registry);
	KeyedStateBackend<Integer> backend = createKeyedStateBackend(registry, numKeyGroups, abstractBackend, dummyEnv);

	final TestRegistryListener registryListener = new TestRegistryListener();
	registry.registerListener(dummyEnv.getJobID(), registryListener);

	// Register state
	ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
	desc.setQueryable("vanilla");

	backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);

	assertTrue(registryListener.registrationName.equals("vanilla"));

	KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, new byte[0]);
	ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);

	// Write the request and wait for the response
	channel.writeInbound(serRequest);

	ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
	buf.skipBytes(4); // skip frame length

	// Verify the response
	assertEquals(MessageType.REQUEST_FAILURE, MessageSerializer.deserializeHeader(buf));
	RequestFailure response = MessageSerializer.deserializeRequestFailure(buf);

	assertTrue(response.getCause().getMessage().contains("RejectedExecutionException"));

	assertEquals(1L, stats.getNumRequests());
	assertEquals(1L, stats.getNumFailed());

	localTestServer.shutdown();
}
 

/**
 * Tests the failure response on a rejected execution, because the query executor has been closed.
 */
@Test
public void testQueryExecutorShutDown() throws Throwable {
	KvStateRegistry registry = new KvStateRegistry();
	AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();

	KvStateServerImpl localTestServer = new KvStateServerImpl(
			InetAddress.getLocalHost().getHostName(),
			Collections.singletonList(0).iterator(),
			1,
			1,
			new KvStateRegistry(),
			new DisabledKvStateRequestStats());

	localTestServer.start();
	localTestServer.shutdown();
	assertTrue(localTestServer.getQueryExecutor().isTerminated());

	MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer =
			new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());

	KvStateServerHandler handler = new KvStateServerHandler(localTestServer, registry, serializer, stats);
	EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);

	int numKeyGroups = 1;
	AbstractStateBackend abstractBackend = new MemoryStateBackend();
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	dummyEnv.setKvStateRegistry(registry);
	KeyedStateBackend<Integer> backend = createKeyedStateBackend(registry, numKeyGroups, abstractBackend, dummyEnv);

	final TestRegistryListener registryListener = new TestRegistryListener();
	registry.registerListener(dummyEnv.getJobID(), registryListener);

	// Register state
	ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
	desc.setQueryable("vanilla");

	backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);

	assertTrue(registryListener.registrationName.equals("vanilla"));

	KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, new byte[0]);
	ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);

	// Write the request and wait for the response
	channel.writeInbound(serRequest);

	ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
	buf.skipBytes(4); // skip frame length

	// Verify the response
	assertEquals(MessageType.REQUEST_FAILURE, MessageSerializer.deserializeHeader(buf));
	RequestFailure response = MessageSerializer.deserializeRequestFailure(buf);
	buf.release();

	assertTrue(response.getCause().getMessage().contains("RejectedExecutionException"));

	assertEquals(1L, stats.getNumRequests());
	assertEquals(1L, stats.getNumFailed());

	localTestServer.shutdown();
}