Java Code Examples for org.apache.flink.api.common.typeutils.base.IntSerializer#INSTANCE

@SuppressWarnings("unchecked")
@Override
protected TupleSerializer<Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>>> createSerializer() {
	return new  TupleSerializer<Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>>>(
			(Class<Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>>>) (Class<?>) Tuple3.class,
			new TypeSerializer[]{
				new TupleSerializer<Tuple2<String, Double>> (
						(Class<Tuple2<String, Double>>) (Class<?>) Tuple2.class,
						new TypeSerializer[]{
								StringSerializer.INSTANCE,
								DoubleSerializer.INSTANCE
				}),
				new TupleSerializer<Tuple2<Long, Long>> (
						(Class<Tuple2<Long, Long>>) (Class<?>) Tuple2.class,
						new TypeSerializer[]{
								LongSerializer.INSTANCE,
								LongSerializer.INSTANCE
				}),
				new TupleSerializer<Tuple2<Integer, Long>> (
						(Class<Tuple2<Integer, Long>>) (Class<?>) Tuple2.class,
						new TypeSerializer[]{
								IntSerializer.INSTANCE,
								LongSerializer.INSTANCE
				})
			});
}
 

@SuppressWarnings("unchecked")
@Override
protected TupleSerializer<Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>>> createSerializer() {
	return new  TupleSerializer<Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>>>(
			(Class<Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>>>) (Class<?>) Tuple3.class,
			new TypeSerializer[]{
				new TupleSerializer<Tuple2<String, Double>> (
						(Class<Tuple2<String, Double>>) (Class<?>) Tuple2.class,
						new TypeSerializer[]{
								StringSerializer.INSTANCE,
								DoubleSerializer.INSTANCE
				}),
				new TupleSerializer<Tuple2<Long, Long>> (
						(Class<Tuple2<Long, Long>>) (Class<?>) Tuple2.class,
						new TypeSerializer[]{
								LongSerializer.INSTANCE,
								LongSerializer.INSTANCE
				}),
				new TupleSerializer<Tuple2<Integer, Long>> (
						(Class<Tuple2<Integer, Long>>) (Class<?>) Tuple2.class,
						new TypeSerializer[]{
								IntSerializer.INSTANCE,
								LongSerializer.INSTANCE
				})
			});
}
 

@SuppressWarnings("rawtypes")
@Override
protected GenericPairComparator<Tuple3<Integer, String, Double>, Tuple4<Integer, Float, Long, Double>> createComparator(boolean ascending) {
	int[] fields1 = new int[]{0, 2};
	int[] fields2 = new int[]{0, 3};
	TypeComparator[] comps1 = new TypeComparator[]{
			new IntComparator(ascending),
			new DoubleComparator(ascending)
	};
	TypeComparator[] comps2 = new TypeComparator[]{
			new IntComparator(ascending),
			new DoubleComparator(ascending)
	};
	TypeSerializer[] sers1 = new TypeSerializer[]{
			IntSerializer.INSTANCE,
			DoubleSerializer.INSTANCE
	};
	TypeSerializer[] sers2= new TypeSerializer[]{
			IntSerializer.INSTANCE,
			DoubleSerializer.INSTANCE
	};
	TypeComparator<Tuple3<Integer, String, Double>> comp1 = new TupleComparator<Tuple3<Integer, String, Double>>(fields1, comps1, sers1);
	TypeComparator<Tuple4<Integer, Float, Long, Double>> comp2 = new TupleComparator<Tuple4<Integer, Float, Long, Double>>(fields2, comps2, sers2);
	return new GenericPairComparator<Tuple3<Integer, String, Double>, Tuple4<Integer, Float, Long, Double>>(comp1, comp2);
}
 

private OneInputStreamOperatorTestHarness<Integer, Object> createTestHarness(
	String topic,
	int maxParallelism,
	int parallelism,
	int subtaskIndex,
	FlinkKafkaProducer.Semantic semantic) throws Exception {
	Properties properties = createProperties();

	FlinkKafkaProducer<Integer> kafkaProducer = new FlinkKafkaProducer<>(
		topic,
		integerKeyedSerializationSchema,
		properties,
		semantic);

	return new OneInputStreamOperatorTestHarness<>(
		new StreamSink<>(kafkaProducer),
		maxParallelism,
		parallelism,
		subtaskIndex,
		IntSerializer.INSTANCE,
		new OperatorID(42, 44));
}
 

/**
 * Verifies deserialization failure cases when reading a serializer from bytes, in the
 * case of a {@link ClassNotFoundException}.
 */
@Test
public void testSerializerSerializationWithClassNotFound() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in),
			new ArtificialCNFExceptionThrowingClassLoader(
				Thread.currentThread().getContextClassLoader(),
				Collections.singleton(IntSerializer.class.getName())),
			true);
	}
	Assert.assertTrue(deserializedSerializer instanceof UnloadableDummyTypeSerializer);

	Assert.assertArrayEquals(
			InstantiationUtil.serializeObject(serializer),
			((UnloadableDummyTypeSerializer<?>) deserializedSerializer).getActualBytes());
}
 

@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{2, 0, 1},
			new TypeComparator[]{
				new DoubleComparator(ascending),
				new IntComparator(ascending),
				new LongComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE, DoubleSerializer.INSTANCE });
}
 

@Test
public void testCheckConcurrencyProblemWhenPerformingCheckpointAsync() throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();
	Environment env = new DummyEnvironment();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);

	ExecutorService executorService = Executors.newScheduledThreadPool(1);
	try {
		long checkpointID = 0;
		List<Future> futureList = new ArrayList();
		for (int i = 0; i < 10; ++i) {
			ValueStateDescriptor<Integer> kvId = new ValueStateDescriptor<>("id" + i, IntSerializer.INSTANCE);
			ValueState<Integer> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
			((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
			backend.setCurrentKey(i);
			state.update(i);

			futureList.add(runSnapshotAsync(executorService,
				backend.snapshot(checkpointID++, System.currentTimeMillis(), streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())));
		}

		for (Future future : futureList) {
			future.get(20, TimeUnit.SECONDS);
		}
	} catch (Exception e) {
		fail();
	} finally {
		backend.dispose();
		executorService.shutdown();
	}
}
 

@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{0, 1, 2},
			new TypeComparator[]{
				new IntComparator(ascending),
				new LongComparator(ascending),
				new DoubleComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE, DoubleSerializer.INSTANCE });
}
 

@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{2, 1},
			new TypeComparator[]{
				new DoubleComparator(ascending),
				new LongComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, DoubleSerializer.INSTANCE, LongSerializer.INSTANCE });
}
 

@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{0, 1},
			new TypeComparator[]{
				new IntComparator(ascending),
				new LongComparator(ascending)
			},
			new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE });
}
 

@Override
protected TupleComparator<Tuple3<Integer, String, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, String, Double>>(
			new int[]{0, 1},
			new TypeComparator[]{
				new IntComparator(ascending),
				new StringComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, StringSerializer.INSTANCE, DoubleSerializer.INSTANCE });
}
 

private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>();
	long timeout = 10L; // 1 milli second

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncFunction,
		timeout,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	harness.setProcessingTime(10L);

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}
}
 

@Test
public void testCheckpointAndRestore() {
	try {
		final int numElements = 10000;

		List<Integer> data = new ArrayList<Integer>(numElements);
		List<Integer> result = new ArrayList<Integer>(numElements);

		for (int i = 0; i < numElements; i++) {
			data.add(i);
		}

		final FromElementsFunction<Integer> source = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
		StreamSource<Integer, FromElementsFunction<Integer>> src = new StreamSource<>(source);
		AbstractStreamOperatorTestHarness<Integer> testHarness =
			new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
		testHarness.open();

		final SourceFunction.SourceContext<Integer> ctx = new ListSourceContext<Integer>(result, 2L);

		final Throwable[] error = new Throwable[1];

		// run the source asynchronously
		Thread runner = new Thread() {
			@Override
			public void run() {
				try {
					source.run(ctx);
				}
				catch (Throwable t) {
					error[0] = t;
				}
			}
		};
		runner.start();

		// wait for a bit
		Thread.sleep(1000);

		// make a checkpoint
		List<Integer> checkpointData = new ArrayList<>(numElements);
		OperatorSubtaskState handles = null;
		synchronized (ctx.getCheckpointLock()) {
			handles = testHarness.snapshot(566, System.currentTimeMillis());
			checkpointData.addAll(result);
		}

		// cancel the source
		source.cancel();
		runner.join();

		// check for errors
		if (error[0] != null) {
			System.err.println("Error in asynchronous source runner");
			error[0].printStackTrace();
			fail("Error in asynchronous source runner");
		}

		final FromElementsFunction<Integer> sourceCopy = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
		StreamSource<Integer, FromElementsFunction<Integer>> srcCopy = new StreamSource<>(sourceCopy);
		AbstractStreamOperatorTestHarness<Integer> testHarnessCopy =
			new AbstractStreamOperatorTestHarness<>(srcCopy, 1, 1, 0);
		testHarnessCopy.setup();
		testHarnessCopy.initializeState(handles);
		testHarnessCopy.open();

		// recovery run
		SourceFunction.SourceContext<Integer> newCtx = new ListSourceContext<>(checkpointData);

		sourceCopy.run(newCtx);

		assertEquals(data, checkpointData);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

TtlFixedLenElemListStateTestContext() {
	super(IntSerializer.INSTANCE);
}
 

TtlReducingStateVerifier() {
	super(new ReducingStateDescriptor<>(
		TtlReducingStateVerifier.class.getSimpleName(),
		(ReduceFunction<Integer>) (value1, value2) -> value1 + value2,
		IntSerializer.INSTANCE));
}
 

private OneInputStreamOperatorTestHarness<Integer, Void> getTestHarness() throws Exception {
	return new OneInputStreamOperatorTestHarness<>(new StreamFlatMap<>(new StatefulFunction()), IntSerializer.INSTANCE);
}
 

/**
 * Tests the failure response on a rejected execution, because the query executor has been closed.
 */
@Test
public void testQueryExecutorShutDown() throws Throwable {
	KvStateRegistry registry = new KvStateRegistry();
	AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();

	KvStateServerImpl localTestServer = new KvStateServerImpl(
			InetAddress.getLocalHost(),
			Collections.singletonList(0).iterator(),
			1,
			1,
			new KvStateRegistry(),
			new DisabledKvStateRequestStats());

	localTestServer.start();
	localTestServer.shutdown();
	assertTrue(localTestServer.getQueryExecutor().isTerminated());

	MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer =
			new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());

	KvStateServerHandler handler = new KvStateServerHandler(localTestServer, registry, serializer, stats);
	EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);

	int numKeyGroups = 1;
	AbstractStateBackend abstractBackend = new MemoryStateBackend();
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	dummyEnv.setKvStateRegistry(registry);
	KeyedStateBackend<Integer> backend = createKeyedStateBackend(registry, numKeyGroups, abstractBackend, dummyEnv);

	final TestRegistryListener registryListener = new TestRegistryListener();
	registry.registerListener(dummyEnv.getJobID(), registryListener);

	// Register state
	ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
	desc.setQueryable("vanilla");

	backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);

	assertTrue(registryListener.registrationName.equals("vanilla"));

	KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, new byte[0]);
	ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);

	// Write the request and wait for the response
	channel.writeInbound(serRequest);

	ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
	buf.skipBytes(4); // skip frame length

	// Verify the response
	assertEquals(MessageType.REQUEST_FAILURE, MessageSerializer.deserializeHeader(buf));
	RequestFailure response = MessageSerializer.deserializeRequestFailure(buf);

	assertTrue(response.getCause().getMessage().contains("RejectedExecutionException"));

	assertEquals(1L, stats.getNumRequests());
	assertEquals(1L, stats.getNumFailed());

	localTestServer.shutdown();
}
 

public static TypeSerializer create(LogicalType type, ExecutionConfig config) {
	switch (type.getTypeRoot()) {
		case BOOLEAN:
			return BooleanSerializer.INSTANCE;
		case TINYINT:
			return ByteSerializer.INSTANCE;
		case SMALLINT:
			return ShortSerializer.INSTANCE;
		case INTEGER:
		case DATE:
		case TIME_WITHOUT_TIME_ZONE:
		case INTERVAL_YEAR_MONTH:
			return IntSerializer.INSTANCE;
		case BIGINT:
		case TIMESTAMP_WITHOUT_TIME_ZONE:
		case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
		case INTERVAL_DAY_TIME:
			return LongSerializer.INSTANCE;
		case FLOAT:
			return FloatSerializer.INSTANCE;
		case DOUBLE:
			return DoubleSerializer.INSTANCE;
		case CHAR:
		case VARCHAR:
			return BinaryStringSerializer.INSTANCE;
		case DECIMAL:
			DecimalType decimalType = (DecimalType) type;
			return new DecimalSerializer(decimalType.getPrecision(), decimalType.getScale());
		case ARRAY:
			return new BaseArraySerializer(((ArrayType) type).getElementType(), config);
		case MAP:
			MapType mapType = (MapType) type;
			return new BaseMapSerializer(mapType.getKeyType(), mapType.getValueType(), config);
		case MULTISET:
			return new BaseMapSerializer(((MultisetType) type).getElementType(), new IntType(), config);
		case ROW:
			RowType rowType = (RowType) type;
			return new BaseRowSerializer(config, rowType);
		case BINARY:
		case VARBINARY:
			return BytePrimitiveArraySerializer.INSTANCE;
		case ANY:
			return new BinaryGenericSerializer(
					((TypeInformationAnyType) type).getTypeInformation().createSerializer(config));
		default:
			throw new RuntimeException("Not support type: " + type);
	}
}
 

private static TypeSerializer<TwoPhaseCommitSinkFunction.State<Integer, String>> intStringStateSerializerSupplier() {
	return new TwoPhaseCommitSinkFunction.StateSerializer<>(IntSerializer.INSTANCE, StringSerializer.INSTANCE);
}
 

@Override
@Nonnull
public TypeSerializer<Integer> getUpdateSerializer() {
	return IntSerializer.INSTANCE;
}