org.apache.flink.api.java.typeutils.runtime.TupleSerializer Java Examples

@SuppressWarnings("unchecked")
@Override
@PublicEvolving
public TupleSerializer<T> createSerializer(ExecutionConfig executionConfig) {
	if (getTypeClass() == Tuple0.class) {
		return (TupleSerializer<T>) Tuple0Serializer.INSTANCE;
	}

	TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[getArity()];
	for (int i = 0; i < types.length; i++) {
		fieldSerializers[i] = types[i].createSerializer(executionConfig);
	}
	
	Class<T> tupleClass = getTypeClass();
	
	return new TupleSerializer<T>(tupleClass, fieldSerializers);
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);

	this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
	
	this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings("unchecked")
@Override
@PublicEvolving
public TupleSerializer<T> createSerializer(ExecutionConfig executionConfig) {
	if (getTypeClass() == Tuple0.class) {
		return (TupleSerializer<T>) Tuple0Serializer.INSTANCE;
	}

	TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[getArity()];
	for (int i = 0; i < types.length; i++) {
		fieldSerializers[i] = types[i].createSerializer(executionConfig);
	}
	
	Class<T> tupleClass = getTypeClass();
	
	return new TupleSerializer<T>(tupleClass, fieldSerializers);
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings("unchecked")
@Override
@PublicEvolving
public TupleSerializer<T> createSerializer(ExecutionConfig executionConfig) {
	if (getTypeClass() == Tuple0.class) {
		return (TupleSerializer<T>) Tuple0Serializer.INSTANCE;
	}

	TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[getArity()];
	for (int i = 0; i < types.length; i++) {
		fieldSerializers[i] = types[i].createSerializer(executionConfig);
	}
	
	Class<T> tupleClass = getTypeClass();
	
	return new TupleSerializer<T>(tupleClass, fieldSerializers);
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 

public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	fields = new Object[((TupleSerializer<IN>) serializer).getArity()];
}
 

@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@SuppressWarnings("unchecked")
@Override
@Nonnull
public TypeSerializer<Tuple2<String, String>> getUpdateSerializer() {
	return new TupleSerializer(
		Tuple2.class, new TypeSerializer[] {StringSerializer.INSTANCE, StringSerializer.INSTANCE});
}
 

public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	fields = new Object[((TupleSerializer<IN>) serializer).getArity()];
}
 

public InPlaceMutableHashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, StringSerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, String>> clazz = (Class<Tuple2<Long, String>>) (Class<?>) Tuple2.class;
	this.serializer = new TupleSerializer<Tuple2<Long, String>>(clazz, fieldSerializers);

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.comparator = new TupleComparator<Tuple2<Long, String>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, String>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, String> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, String> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 

public HashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, BytePrimitiveArraySerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, byte[]>> clazz = (Class<Tuple2<Long, byte[]>>) (Class<?>) Tuple2.class;
	this.buildSerializer = new TupleSerializer<Tuple2<Long, byte[]>>(clazz, fieldSerializers);
	
	this.probeSerializer = LongSerializer.INSTANCE;

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.buildComparator = new TupleComparator<Tuple2<Long, byte[]>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, byte[]>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, byte[]> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, byte[]> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 

public HashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, BytePrimitiveArraySerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, byte[]>> clazz = (Class<Tuple2<Long, byte[]>>) (Class<?>) Tuple2.class;
	this.buildSerializer = new TupleSerializer<Tuple2<Long, byte[]>>(clazz, fieldSerializers);
	
	this.probeSerializer = LongSerializer.INSTANCE;

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.buildComparator = new TupleComparator<Tuple2<Long, byte[]>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, byte[]>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, byte[]> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, byte[]> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState == null,
		"The " + getClass().getSimpleName() + " has already been initialized.");

	this.checkpointedState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>(
			"arrow-source-state",
			new TupleSerializer<>(
				(Class<Tuple2<Integer, Integer>>) (Class<?>) Tuple2.class,
				new TypeSerializer[]{IntSerializer.INSTANCE, IntSerializer.INSTANCE})
		)
	);

	this.indexesToEmit = new ArrayDeque<>();
	if (context.isRestored()) {
		// upon restoring
		for (Tuple2<Integer, Integer> v : this.checkpointedState.get()) {
			this.indexesToEmit.add(v);
		}
		LOG.info("Subtask {} restored state: {}.", getRuntimeContext().getIndexOfThisSubtask(), indexesToEmit);
	} else {
		// the first time the job is executed
		final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks();
		final int taskIdx = getRuntimeContext().getIndexOfThisSubtask();

		for (int i = taskIdx; i < arrowData.length; i += stepSize) {
			this.indexesToEmit.add(Tuple2.of(i, 0));
		}
		LOG.info("Subtask {} has no restore state, initialized with {}.", taskIdx, indexesToEmit);
	}
}
 

/**
 * Before using an explicit TypeSerializer for the partition state the {@link
 * FlinkKafkaConsumerBase} was creating a serializer using a {@link TypeHint}. Here, we verify
 * that the two methods create compatible serializers.
 */
@Test
public void testExplicitStateSerializerCompatibility() throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();

	Tuple2<KafkaTopicPartition, Long> tuple =
			new Tuple2<>(new KafkaTopicPartition("dummy", 0), 42L);

	// This is how the KafkaConsumerBase used to create the TypeSerializer
	TypeInformation<Tuple2<KafkaTopicPartition, Long>> originalTypeHintTypeInfo =
			new TypeHint<Tuple2<KafkaTopicPartition, Long>>() {}.getTypeInfo();
	TypeSerializer<Tuple2<KafkaTopicPartition, Long>> serializerFromTypeHint =
			originalTypeHintTypeInfo.createSerializer(executionConfig);
	byte[] bytes = InstantiationUtil.serializeToByteArray(serializerFromTypeHint, tuple);

	// Directly use the Consumer to create the TypeSerializer (using the new method)
	TupleSerializer<Tuple2<KafkaTopicPartition, Long>> kafkaConsumerSerializer =
			FlinkKafkaConsumerBase.createStateSerializer(executionConfig);
	Tuple2<KafkaTopicPartition, Long> actualTuple =
			InstantiationUtil.deserializeFromByteArray(kafkaConsumerSerializer, bytes);

	Assert.assertEquals(
			"Explicit Serializer is not compatible with previous method of creating Serializer using TypeHint.",
			tuple,
			actualTuple
	);
}
 

/**
 * Creates state serializer for kafka topic partition to offset tuple.
 * Using of the explicit state serializer with KryoSerializer is needed because otherwise
 * users cannot use 'disableGenericTypes' properties with KafkaConsumer.
 */
@VisibleForTesting
static TupleSerializer<Tuple2<KafkaTopicPartition, Long>> createStateSerializer(ExecutionConfig executionConfig) {
	// explicit serializer will keep the compatibility with GenericTypeInformation and allow to disableGenericTypes for users
	TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[]{
		new KryoSerializer<>(KafkaTopicPartition.class, executionConfig),
		LongSerializer.INSTANCE
	};
	@SuppressWarnings("unchecked")
	Class<Tuple2<KafkaTopicPartition, Long>> tupleClass = (Class<Tuple2<KafkaTopicPartition, Long>>) (Class<?>) Tuple2.class;
	return new TupleSerializer<>(tupleClass, fieldSerializers);
}
 

@SuppressWarnings("unchecked")
@Override
@Nonnull
public TypeSerializer<Tuple2<String, String>> getUpdateSerializer() {
	return new TupleSerializer(
		Tuple2.class, new TypeSerializer[] {StringSerializer.INSTANCE, StringSerializer.INSTANCE});
}
 

@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@SuppressWarnings("unchecked")
@Override
@Nonnull
public TypeSerializer<Tuple2<String, String>> getUpdateSerializer() {
	return new TupleSerializer(
		Tuple2.class, new TypeSerializer[] {StringSerializer.INSTANCE, StringSerializer.INSTANCE});
}
 

public InPlaceMutableHashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, StringSerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, String>> clazz = (Class<Tuple2<Long, String>>) (Class<?>) Tuple2.class;
	this.serializer = new TupleSerializer<Tuple2<Long, String>>(clazz, fieldSerializers);

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.comparator = new TupleComparator<Tuple2<Long, String>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, String>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, String> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, String> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 

public InPlaceMutableHashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, StringSerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, String>> clazz = (Class<Tuple2<Long, String>>) (Class<?>) Tuple2.class;
	this.serializer = new TupleSerializer<Tuple2<Long, String>>(clazz, fieldSerializers);

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.comparator = new TupleComparator<Tuple2<Long, String>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, String>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, String> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, String> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 

public HashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, BytePrimitiveArraySerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, byte[]>> clazz = (Class<Tuple2<Long, byte[]>>) (Class<?>) Tuple2.class;
	this.buildSerializer = new TupleSerializer<Tuple2<Long, byte[]>>(clazz, fieldSerializers);
	
	this.probeSerializer = LongSerializer.INSTANCE;

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.buildComparator = new TupleComparator<Tuple2<Long, byte[]>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, byte[]>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, byte[]> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, byte[]> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 

public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	fields = new Object[((TupleSerializer<IN>) serializer).getArity()];
}
 

@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Override
	public void open() throws Exception {
		super.open();

		this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
		timestampedCollector = new TimestampedCollector<>(output);

		internalTimerService =
				getInternalTimerService("window-timers", windowSerializer, this);

		triggerContext = new Context(null, null);
		processContext = new WindowContext(null);

		windowAssignerContext = new WindowAssigner.WindowAssignerContext() {
			@Override
			public long getCurrentProcessingTime() {
				return internalTimerService.currentProcessingTime();
			}
		};

		// create (or restore) the state that hold the actual window contents
		// NOTE - the state may be null in the case of the overriding evicting window operator
		if (windowStateDescriptor != null) {
			windowState = (InternalAppendingState<K, W, IN, ACC, ACC>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
		}

		// create the typed and helper states for merging windows
		if (windowAssigner instanceof MergingWindowAssigner) {

			// store a typed reference for the state of merging windows - sanity check
			if (windowState instanceof InternalMergingState) {
				windowMergingState = (InternalMergingState<K, W, IN, ACC, ACC>) windowState;
			}
			// TODO this sanity check should be here, but is prevented by an incorrect test (pending validation)
			// TODO see WindowOperatorTest.testCleanupTimerWithEmptyFoldingStateForSessionWindows()
			// TODO activate the sanity check once resolved
//			else if (windowState != null) {
//				throw new IllegalStateException(
//						"The window uses a merging assigner, but the window state is not mergeable.");
//			}

			@SuppressWarnings("unchecked")
			final Class<Tuple2<W, W>> typedTuple = (Class<Tuple2<W, W>>) (Class<?>) Tuple2.class;

			final TupleSerializer<Tuple2<W, W>> tupleSerializer = new TupleSerializer<>(
					typedTuple,
					new TypeSerializer[] {windowSerializer, windowSerializer});

			final ListStateDescriptor<Tuple2<W, W>> mergingSetsStateDescriptor =
					new ListStateDescriptor<>("merging-window-set", tupleSerializer);

			// get the state that stores the merging sets
			mergingSetsState = (InternalListState<K, VoidNamespace, Tuple2<W, W>>)
					getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, mergingSetsStateDescriptor);
			mergingSetsState.setCurrentNamespace(VoidNamespace.INSTANCE);
		}
	}
 

@Override
protected abstract TupleSerializer<T> createSerializer();
 

@Override
protected abstract TupleSerializer<T> createSerializer();