Java Code Examples for org.apache.flink.table.api.TableSchema#getFieldDataTypes()

public HiveTableOutputFormat(JobConf jobConf, ObjectPath tablePath, CatalogTable table, HiveTablePartition hiveTablePartition,
							Properties tableProperties, boolean overwrite) {
	super(jobConf.getCredentials());

	Preconditions.checkNotNull(table, "table cannot be null");
	Preconditions.checkNotNull(hiveTablePartition, "HiveTablePartition cannot be null");
	Preconditions.checkNotNull(tableProperties, "Table properties cannot be null");

	HadoopUtils.mergeHadoopConf(jobConf);
	this.jobConf = jobConf;
	this.tablePath = tablePath;
	this.partitionColumns = table.getPartitionKeys();
	TableSchema tableSchema = table.getSchema();
	this.fieldNames = tableSchema.getFieldNames();
	this.fieldTypes = tableSchema.getFieldDataTypes();
	this.hiveTablePartition = hiveTablePartition;
	this.tableProperties = tableProperties;
	this.overwrite = overwrite;
	isPartitioned = partitionColumns != null && !partitionColumns.isEmpty();
	isDynamicPartition = isPartitioned && partitionColumns.size() > hiveTablePartition.getPartitionSpec().size();
	hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
}
 

/**
 * Convert time attributes (proc time / event time) to normal timestamps,
 * and return a new {@link TableSchema}.
 */
static TableSchema convertTimeAttributeToRegularTimestamp(TableSchema tableSchema) {
	DataType[] oldTypes = tableSchema.getFieldDataTypes();
	String[] oldNames = tableSchema.getFieldNames();

	TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < tableSchema.getFieldCount(); i++) {
		DataType fieldType = oldTypes[i];
		String fieldName = oldNames[i];
		if (fieldType.getLogicalType() instanceof TimestampType) {
			TimestampType timestampType = (TimestampType) fieldType.getLogicalType();
			if (!timestampType.getKind().equals(TimestampKind.REGULAR)) {
				// converts `TIME ATTRIBUTE(ROWTIME)`/`TIME ATTRIBUTE(PROCTIME)` to `TIMESTAMP`
				builder.field(fieldName, Types.SQL_TIMESTAMP);
				continue;
			}
		}
		builder.field(fieldName, fieldType);
	}
	return builder.build();
}
 

@Test
public void testCheckPhysicalLogicalTypeCompatible() {
	TableSchema tableSchema = TableSchema.builder()
							.field("a", DataTypes.VARCHAR(2))
							.field("b", DataTypes.DECIMAL(20, 2))
							.build();
	TableSink tableSink = new TestTableSink(tableSchema);
	LegacyTypeInformationType legacyDataType = (LegacyTypeInformationType) tableSink.getConsumedDataType()
													.getLogicalType();
	TypeInformation legacyTypeInfo = ((TupleTypeInfo) legacyDataType.getTypeInformation()).getTypeAt(1);
	DataType physicalType = TypeConversions.fromLegacyInfoToDataType(legacyTypeInfo);
	TableSchema physicSchema = DataTypeUtils.expandCompositeTypeToSchema(physicalType);
	DataType[] logicalDataTypes = tableSchema.getFieldDataTypes();
	DataType[] physicalDataTypes = physicSchema.getFieldDataTypes();
	for (int i = 0; i < logicalDataTypes.length; i++) {
		TypeMappingUtils.checkPhysicalLogicalTypeCompatible(
				physicalDataTypes[i].getLogicalType(),
				logicalDataTypes[i].getLogicalType(),
				"physicalField",
				"logicalField",
				false);
	}
}
 

@Override
public DataType getProducedDataType() {
    TableSchema fullSchema = getTableSchema();
    DataType type;
    if (projectedFields == null) {
        type = fullSchema.toRowDataType();
    } else {
        String[] fullNames = fullSchema.getFieldNames();
        DataType[] fullTypes = fullSchema.getFieldDataTypes();
        type = TableSchema.builder().fields(
            Arrays.stream(projectedFields).mapToObj(i -> fullNames[i]).toArray(String[]::new),
            Arrays.stream(projectedFields).mapToObj(i -> fullTypes[i]).toArray(DataType[]::new))
            .build().toRowDataType();
    }
    return type.bridgedTo(BaseRow.class);
}
 

private JdbcTableSource(
	JdbcOptions options, JdbcReadOptions readOptions, JdbcLookupOptions lookupOptions,
	TableSchema schema, int[] selectFields) {
	this.options = options;
	this.readOptions = readOptions;
	this.lookupOptions = lookupOptions;
	this.schema = schema;

	this.selectFields = selectFields;

	final DataType[] schemaDataTypes = schema.getFieldDataTypes();
	final String[] schemaFieldNames = schema.getFieldNames();
	if (selectFields != null) {
		DataType[] dataTypes = new DataType[selectFields.length];
		String[] fieldNames = new String[selectFields.length];
		for (int i = 0; i < selectFields.length; i++) {
			dataTypes[i] = schemaDataTypes[selectFields[i]];
			fieldNames[i] = schemaFieldNames[selectFields[i]];
		}
		this.producedDataType =
				TableSchema.builder().fields(fieldNames, dataTypes).build().toRowDataType();
	} else {
		this.producedDataType = schema.toRowDataType();
	}
}
 

/**
 * Convert time attributes (proc time / event time) to regular timestamp
 * and build a new {@link TableSchema}.
 */
public static TableSchema convertTimeAttributeToRegularTimestamp(TableSchema tableSchema) {
	DataType[] dataTypes = tableSchema.getFieldDataTypes();
	String[] oldNames = tableSchema.getFieldNames();

	TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < tableSchema.getFieldCount(); i++) {
		DataType fieldType = dataTypes[i];
		String fieldName = oldNames[i];
		if (fieldType.getLogicalType() instanceof TimestampType) {
			TimestampType timestampType = (TimestampType) fieldType.getLogicalType();
			if (!timestampType.getKind().equals(TimestampKind.REGULAR)) {
				// converts `TIME ATTRIBUTE(ROWTIME)`/`TIME ATTRIBUTE(PROCTIME)` to `TIMESTAMP(3)`
				builder.field(fieldName, DataTypes.TIMESTAMP(3));
				continue;
			}
		}
		builder.field(fieldName, fieldType);
	}
	return builder.build();
}
 

/**
 * Change to default conversion class and build a new {@link TableSchema}.
 */
public static TableSchema changeDefaultConversionClass(TableSchema tableSchema) {
	DataType[] oldTypes = tableSchema.getFieldDataTypes();
	String[] fieldNames = tableSchema.getFieldNames();

	TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < tableSchema.getFieldCount(); i++) {
		DataType fieldType = LogicalTypeDataTypeConverter.fromLogicalTypeToDataType(
				LogicalTypeDataTypeConverter.fromDataTypeToLogicalType(oldTypes[i]));
		builder.field(fieldNames[i], fieldType);
	}
	return builder.build();
}
 

private TableSchema getProducedTableSchema() {
	TableSchema fullSchema = getTableSchema();
	if (projectedFields == null) {
		return fullSchema;
	} else {
		String[] fullNames = fullSchema.getFieldNames();
		DataType[] fullTypes = fullSchema.getFieldDataTypes();
		return TableSchema.builder().fields(
				Arrays.stream(projectedFields).mapToObj(i -> fullNames[i]).toArray(String[]::new),
				Arrays.stream(projectedFields).mapToObj(i -> fullTypes[i]).toArray(DataType[]::new)).build();
	}
}
 

private static TableSchema removeTimeAttributes(TableSchema schema) {
	final TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < schema.getFieldCount(); i++) {
		final DataType dataType = schema.getFieldDataTypes()[i];
		final DataType convertedType = DataTypeUtils.replaceLogicalType(
			dataType,
			LogicalTypeUtils.removeTimeAttributes(dataType.getLogicalType()));
		builder.field(schema.getFieldNames()[i], convertedType);
	}
	return builder.build();
}
 

/**
 * Create Hive columns from Flink TableSchema.
 */
public static List<FieldSchema> createHiveColumns(TableSchema schema) {
	String[] fieldNames = schema.getFieldNames();
	DataType[] fieldTypes = schema.getFieldDataTypes();

	List<FieldSchema> columns = new ArrayList<>(fieldNames.length);

	for (int i = 0; i < fieldNames.length; i++) {
		columns.add(
			new FieldSchema(fieldNames[i], HiveTypeUtil.toHiveTypeInfo(fieldTypes[i], true).getTypeName(), null));
	}

	return columns;
}
 

/**
 * Creates FlinkPravegaOutputFormat based on the given table schema and current builder state.
 * @param tableSchema the schema of the sink table
 */
public FlinkPravegaOutputFormat<Row> createOutputFormat(TableSchema tableSchema) {
    Preconditions.checkState(routingKeyFieldName != null, "The routing key field must be provided.");
    Preconditions.checkState(serializationSchema != null, "The serializationSchema must be provided.");
    PravegaEventRouter<Row> eventRouter = new FlinkPravegaTableSink.RowBasedRouter(routingKeyFieldName, tableSchema.getFieldNames(), tableSchema.getFieldDataTypes());
    return new FlinkPravegaOutputFormat<>(
            getPravegaConfig().getClientConfig(),
            resolveStream(),
            serializationSchema,
            eventRouter
    );
}
 

/**
 * Create Hive columns from Flink TableSchema.
 */
public static List<FieldSchema> createHiveColumns(TableSchema schema) {
	String[] fieldNames = schema.getFieldNames();
	DataType[] fieldTypes = schema.getFieldDataTypes();

	List<FieldSchema> columns = new ArrayList<>(fieldNames.length);

	for (int i = 0; i < fieldNames.length; i++) {
		columns.add(
			new FieldSchema(fieldNames[i], HiveTypeUtil.toHiveTypeName(fieldTypes[i]), null));
	}

	return columns;
}
 

@Override
public Table getBatchTable(String tableName, Params parameter, Long sessionId) throws Exception {
    ExecutionEnvironment env = MLEnvironmentFactory.get(sessionId).getExecutionEnvironment();
    HiveBatchSource hiveTableSource = getHiveBatchSource(tableName, parameter);
    DataSet<BaseRow> dataSet = hiveTableSource.getDataSet(env);
    TableSchema schema = hiveTableSource.getTableSchema();
    final DataType[] dataTypes = schema.getFieldDataTypes();
    DataSet<Row> rows = dataSet.map(new BaseRowToRow(dataTypes));
    Table tbl = DataSetConversionUtil.toTable(sessionId, rows, schema);
    if (getPartitionCols(tableName).size() > 0) { // remove static partition columns
        String[] fieldNames = getColNames(tableName);
        tbl = tbl.select(Strings.join(fieldNames, ","));
    }
    return tbl;
}
 

public QueryResult getFlinkQueryResult(Entry<TableSchema, List<Row>> sparkData) throws Exception {
    if (sparkData == null) {
        return new QueryResult(new ArrayList<>(), new ArrayList<>());
    }
    TableSchema tableSchema = sparkData.getKey();
    if (tableSchema == null || tableSchema.getFieldDataTypes().length != tableSchema.getFieldNames().length) {
        throw new SparkException("collect data error");
    }
    org.apache.flink.table.types.DataType[] fieldDataTypes = tableSchema.getFieldDataTypes();
    String[] fieldNames = tableSchema.getFieldNames();
    List<Row> value = sparkData.getValue();
    List<Object> data = new ArrayList<>();
    List<ColumnMetaData> meta = new ArrayList<>();
    value.stream().forEach(column -> {
        Object[] objects = new Object[column.getArity()];
        for (int i = 0; i < column.getArity(); i++) {
            objects[i] = column.getField(i);
        }
        data.add(Arrays.asList(objects));
    });
    for (int index = 0; index < fieldNames.length; index++) {
        ScalarType columnType = getColumnType2(fieldDataTypes[index]);
        meta.add(new ColumnMetaData(index, false, true, false, false,
            1, true, -1, fieldNames[index], fieldNames[index], null, -1, -1, null, null,
            columnType, true, false, false, columnType.columnClassName()));
    }
    return new QueryResult(meta, data);
}
 

private static TableSchema removeTimeAttributes(TableSchema schema) {
	final TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < schema.getFieldCount(); i++) {
		final DataType dataType = schema.getFieldDataTypes()[i];
		final DataType convertedType = DataTypeUtils.replaceLogicalType(
			dataType,
			LogicalTypeUtils.removeTimeAttributes(dataType.getLogicalType()));
		builder.field(schema.getFieldNames()[i], convertedType);
	}
	return builder.build();
}
 

public Mapper(TableSchema dataSchema, Params params) {
	this.dataFieldNames = dataSchema.getFieldNames();
	this.dataFieldTypes = dataSchema.getFieldDataTypes();
	this.params = (null == params) ? new Params() : params.clone();
}
 

/**
 * Resolve the computed column's type for the given schema.
 *
 * @param tableSchema Table schema to derive table field names and data types
 * @return the resolved TableSchema
 */
public TableSchema resolve(TableSchema tableSchema) {
	final String rowtime;
	if (!tableSchema.getWatermarkSpecs().isEmpty()) {
		// TODO: [FLINK-14473] we only support top-level rowtime attribute right now
		rowtime = tableSchema.getWatermarkSpecs().get(0).getRowtimeAttribute();
		if (rowtime.contains(".")) {
			throw new ValidationException(
					String.format("Nested field '%s' as rowtime attribute is not supported right now.", rowtime));
		}
	} else {
		rowtime = null;
	}

	String[] fieldNames = tableSchema.getFieldNames();
	DataType[] fieldTypes = tableSchema.getFieldDataTypes();

	TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < tableSchema.getFieldCount(); ++i) {
		TableColumn tableColumn = tableSchema.getTableColumns().get(i);
		DataType fieldType = fieldTypes[i];

		if (tableColumn.isGenerated()) {
			fieldType = resolveExpressionDataType(tableColumn.getExpr().get(), tableSchema);
			if (isProctime(fieldType)) {
				if (fieldNames[i].equals(rowtime)) {
					throw new TableException("Watermark can not be defined for a processing time attribute column.");
				}
			}
		}

		if (isStreamingMode && fieldNames[i].equals(rowtime)) {
			TimestampType originalType = (TimestampType) fieldType.getLogicalType();
			LogicalType rowtimeType = new TimestampType(
					originalType.isNullable(),
					TimestampKind.ROWTIME,
					originalType.getPrecision());
			fieldType = TypeConversions.fromLogicalToDataType(rowtimeType);
		}

		if (tableColumn.isGenerated()) {
			builder.field(fieldNames[i], fieldType, tableColumn.getExpr().get());
		} else {
			builder.field(fieldNames[i], fieldType);
		}
	}

	tableSchema.getWatermarkSpecs().forEach(builder::watermark);
	tableSchema.getPrimaryKey().ifPresent(
			pk -> builder.primaryKey(pk.getName(), pk.getColumns().toArray(new String[0])));
	return builder.build();
}
 

/** Convert CREATE VIEW statement. */
private Operation convertCreateView(SqlCreateView sqlCreateView) {
	final SqlNode query = sqlCreateView.getQuery();
	final SqlNodeList fieldList = sqlCreateView.getFieldList();

	SqlNode validateQuery = flinkPlanner.validate(query);
	PlannerQueryOperation operation = toQueryOperation(flinkPlanner, validateQuery);
	TableSchema schema = operation.getTableSchema();

	// the view column list in CREATE VIEW is optional, if it's not empty, we should update
	// the column name with the names in view column list.
	if (!fieldList.getList().isEmpty()) {
		// alias column names
		String[] inputFieldNames = schema.getFieldNames();
		String[] aliasFieldNames = fieldList.getList().stream()
				.map(SqlNode::toString)
				.toArray(String[]::new);

		if (inputFieldNames.length != aliasFieldNames.length) {
			throw new SqlConversionException(String.format(
					"VIEW definition and input fields not match:\n\tDef fields: %s.\n\tInput fields: %s.",
					Arrays.toString(aliasFieldNames), Arrays.toString(inputFieldNames)));
		}

		DataType[] inputFieldTypes = schema.getFieldDataTypes();
		schema = TableSchema.builder().fields(aliasFieldNames, inputFieldTypes).build();
	}

	String originalQuery = getQuotedSqlString(query);
	String expandedQuery = getQuotedSqlString(validateQuery);
	String comment = sqlCreateView.getComment().map(c -> c.getNlsString().getValue()).orElse(null);
	CatalogView catalogView = new CatalogViewImpl(originalQuery,
			expandedQuery,
			schema,
			Collections.emptyMap(),
			comment);

	UnresolvedIdentifier unresolvedIdentifier = UnresolvedIdentifier.of(sqlCreateView.fullViewName());
	ObjectIdentifier identifier = catalogManager.qualifyIdentifier(unresolvedIdentifier);

	return new CreateViewOperation(
			identifier,
			catalogView,
			sqlCreateView.isIfNotExists(),
			sqlCreateView.isTemporary());
}
 

/**
 * Adds a table schema under the given key.
 */
public void putTableSchema(String key, TableSchema schema) {
	checkNotNull(key);
	checkNotNull(schema);

	final String[] fieldNames = schema.getFieldNames();
	final DataType[] fieldTypes = schema.getFieldDataTypes();
	final String[] fieldExpressions = schema.getTableColumns().stream()
		.map(column -> column.getExpr().orElse(null))
		.toArray(String[]::new);

	final List<List<String>> values = new ArrayList<>();
	for (int i = 0; i < schema.getFieldCount(); i++) {
		values.add(
			Arrays.asList(
				fieldNames[i],
				fieldTypes[i].getLogicalType().asSerializableString(),
				fieldExpressions[i]));
	}

	putIndexedOptionalProperties(
		key,
		Arrays.asList(NAME, DATA_TYPE, EXPR),
		values);

	if (!schema.getWatermarkSpecs().isEmpty()) {
		final List<List<String>> watermarkValues = new ArrayList<>();
		for (WatermarkSpec spec : schema.getWatermarkSpecs()) {
			watermarkValues.add(Arrays.asList(
				spec.getRowtimeAttribute(),
				spec.getWatermarkExpr(),
				spec.getWatermarkExprOutputType().getLogicalType().asSerializableString()));
		}
		putIndexedFixedProperties(
			key + '.' + WATERMARK,
			Arrays.asList(WATERMARK_ROWTIME, WATERMARK_STRATEGY_EXPR, WATERMARK_STRATEGY_DATA_TYPE),
			watermarkValues);
	}

	schema.getPrimaryKey().ifPresent(pk -> {
		putString(key + '.' + PRIMARY_KEY_NAME, pk.getName());
		putString(key + '.' + PRIMARY_KEY_COLUMNS, String.join(",", pk.getColumns()));
	});
}
 

/**
 * Constructs a ModelMapper.
 *
 * @param modelSchema The schema of the model rows passed to {@link #loadModel(List)}.
 * @param dataSchema The schema of the input data rows.
 * @param params The parameters of this ModelMapper.
 */
public ModelMapper(TableSchema modelSchema, TableSchema dataSchema, Params params) {
	super(dataSchema, params);
	this.modelFieldNames = modelSchema.getFieldNames();
	this.modelFieldTypes = modelSchema.getFieldDataTypes();
}