org.apache.flink.api.java.typeutils.runtime.RowSerializer Java Examples

@Override
public PythonFunctionRunner<Row> createPythonFunctionRunner(
		FnDataReceiver<byte[]> resultReceiver,
		PythonEnvironmentManager pythonEnvironmentManager,
		Map<String, String> jobOptions) {
	return new PassThroughPythonScalarFunctionRunner<Row>(
		getRuntimeContext().getTaskName(),
		resultReceiver,
		scalarFunctions,
		PythonTestUtils.createTestEnvironmentManager(),
		userDefinedFunctionInputType,
		userDefinedFunctionOutputType,
		jobOptions,
		PythonTestUtils.createMockFlinkMetricContainer()) {
		@Override
		public TypeSerializer<Row> getInputTypeSerializer() {
			return (RowSerializer) PythonTypeUtils.toFlinkTypeSerializer(getInputType());
		}
	};
}
 

@Override
public TypeSerializer<Row> createSerializer(ExecutionConfig config) {
	int len = getArity();
	TypeSerializer<?>[] fieldSerializers = new TypeSerializer[len];
	for (int i = 0; i < len; i++) {
		fieldSerializers[i] = types[i].createSerializer(config);
	}
	return new RowSerializer(fieldSerializers);
}
 

/**
 * Creates a serializer for the old {@link Row} format before Flink 1.11.
 *
 * <p>The serialization format has changed from 1.10 to 1.11 and added {@link Row#getKind()}.
 */
@Deprecated
public TypeSerializer<Row> createLegacySerializer(ExecutionConfig config) {
	int len = getArity();
	TypeSerializer<?>[] fieldSerializers = new TypeSerializer[len];
	for (int i = 0; i < len; i++) {
		fieldSerializers[i] = types[i].createSerializer(config);
	}
	return new RowSerializer(fieldSerializers, true);
}
 

@Override
public TypeSerializer<Row> createSerializer(ExecutionConfig config) {
	int len = getArity();
	TypeSerializer<?>[] fieldSerializers = new TypeSerializer[len];
	for (int i = 0; i < len; i++) {
		fieldSerializers[i] = types[i].createSerializer(config);
	}
	return new RowSerializer(fieldSerializers);
}
 

private AbstractGeneralPythonScalarFunctionRunner<Row> createUDFRunner(
	JobBundleFactory jobBundleFactory, FnDataReceiver<byte[]> receiver) {
	PythonFunctionInfo[] pythonFunctionInfos = new PythonFunctionInfo[] {
		new PythonFunctionInfo(
			DummyPythonFunction.INSTANCE,
			new Integer[]{0})
	};

	RowType rowType = new RowType(Collections.singletonList(new RowType.RowField("f1", new BigIntType())));

	final PythonEnvironmentManager environmentManager = createTestEnvironmentManager();

	return new PassThroughPythonScalarFunctionRunner<Row>(
		"testPythonRunner",
		receiver,
		pythonFunctionInfos,
		environmentManager,
		rowType,
		rowType,
		Collections.emptyMap(),
		jobBundleFactory,
		PythonTestUtils.createMockFlinkMetricContainer()) {
		@Override
		public TypeSerializer<Row> getInputTypeSerializer() {
			return (RowSerializer) PythonTypeUtils.toFlinkTypeSerializer(getInputType());
		}
	};
}
 

@Test
public void testInputOutputDataTypeConstructedProperlyForChainedUDFs() throws Exception {
	final PythonScalarFunctionRunner runner = (PythonScalarFunctionRunner) createChainedUDFRunner();

	// check input TypeSerializer
	TypeSerializer inputTypeSerializer = runner.getInputTypeSerializer();
	assertEquals(5, ((RowSerializer) inputTypeSerializer).getArity());
}
 

@Test
public void testInputOutputDataTypeConstructedProperlyForMultipleUDFs() throws Exception {
	final PythonScalarFunctionRunner runner = (PythonScalarFunctionRunner) createMultipleUDFRunner();

	// check input TypeSerializer
	TypeSerializer inputTypeSerializer = runner.getInputTypeSerializer();
	assertEquals(3, ((RowSerializer) inputTypeSerializer).getArity());
}
 

@Test
public void testInputOutputDataTypeConstructedProperlyForSingleUDF() throws Exception {
	final PythonScalarFunctionRunner runner = (PythonScalarFunctionRunner) createSingleUDFRunner();

	// check input TypeSerializer
	TypeSerializer inputTypeSerializer = runner.getInputTypeSerializer();
	assertEquals(1, ((RowSerializer) inputTypeSerializer).getArity());
}
 

@Test
public void testInputOutputDataTypeConstructedProperlyForSingleUDTF() throws Exception {
	final AbstractPythonTableFunctionRunner<Row> runner = createUDTFRunner();

	// check input TypeSerializer
	TypeSerializer inputTypeSerializer = runner.getInputTypeSerializer();
	assertTrue(inputTypeSerializer instanceof RowSerializer);

	assertEquals(1, ((RowSerializer) inputTypeSerializer).getArity());
}
 

@Test
public void testLogicalTypeToFlinkTypeSerializer() {
	List<RowType.RowField> rowFields = new ArrayList<>();
	rowFields.add(new RowType.RowField("f1", new BigIntType()));
	RowType rowType = new RowType(rowFields);
	TypeSerializer rowSerializer = PythonTypeUtils.toFlinkTypeSerializer(rowType);
	assertTrue(rowSerializer instanceof RowSerializer);

	assertEquals(1, ((RowSerializer) rowSerializer).getArity());
}
 

@Override
public TypeSerializer visit(RowType rowType) {
	final TypeSerializer[] fieldTypeSerializers = rowType.getFields()
		.stream()
		.map(f -> f.getType().accept(this))
		.toArray(TypeSerializer[]::new);
	return new RowSerializer(fieldTypeSerializers, true);
}
 

public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	arity = ((RowSerializer) serializer).getArity();
	fields = new Object[arity];
}
 

@Override
public TypeSerializer<Row> createSerializer(ExecutionConfig config) {
	int len = getArity();
	TypeSerializer<?>[] fieldSerializers = new TypeSerializer[len];
	for (int i = 0; i < len; i++) {
		fieldSerializers[i] = types[i].createSerializer(config);
	}
	return new RowSerializer(fieldSerializers);
}
 

public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	arity = ((RowSerializer) serializer).getArity();
	fields = new Object[arity];
}
 

public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	arity = ((RowSerializer) serializer).getArity();
	fields = new Object[arity];
}
 

public RowArrowSourceFunctionTest() {
	super(VectorSchemaRoot.create(ArrowUtils.toArrowSchema(rowType), allocator),
		new RowSerializer(new TypeSerializer[]{StringSerializer.INSTANCE}, true),
		Comparator.comparing(o -> (String) (o.getField(0))));
}
 

@Override
@SuppressWarnings("unchecked")
public RowSerializer getInputTypeSerializer() {
	return (RowSerializer) PythonTypeUtils.toFlinkTypeSerializer(getInputType());
}
 

@Override
public TypeSerializer<Row> getInputTypeSerializer() {
	return (RowSerializer) PythonTypeUtils.toFlinkTypeSerializer(getInputType());
}