Java Code Examples for org.apache.flink.api.java.typeutils.RowTypeInfo#getFieldNames()

/**
 * Creates RowRowRecordParser from output RowTypeInfo and selected series in the RowRecord. The row field "time"
 * will be used to store the timestamp value. The other row fields store the values of the same field names of
 * the RowRecord.
 *
 * @param outputRowTypeInfo The RowTypeInfo of the output row.
 * @param selectedSeries The selected series in the RowRecord.
 * @return The RowRowRecordParser.
 */
public static RowRowRecordParser create(RowTypeInfo outputRowTypeInfo, List<Path> selectedSeries) {
	List<String> selectedSeriesNames = selectedSeries.stream().map(Path::toString).collect(Collectors.toList());
	String[] rowFieldNames = outputRowTypeInfo.getFieldNames();
	int[] indexMapping = new int[outputRowTypeInfo.getArity()];
	for (int i = 0; i < outputRowTypeInfo.getArity(); i++) {
		if (!QueryConstant.RESERVED_TIME.equals(rowFieldNames[i])) {
			int index = selectedSeriesNames.indexOf(rowFieldNames[i]);
			if (index >= 0) {
				indexMapping[i] = index;
			} else {
				throw new IllegalArgumentException(rowFieldNames[i] + " is not found in selected series.");
			}
		} else {
			// marked as timestamp field.
			indexMapping[i] = -1;
		}
	}
	return new RowRowRecordParser(indexMapping, outputRowTypeInfo);
}
 

private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) {
	if (reuse == null) {
		reuse = mapper.createObjectNode();
	}
	final String[] fieldNames = info.getFieldNames();
	final TypeInformation<?>[] fieldTypes = info.getFieldTypes();

	// validate the row
	if (row.getArity() != fieldNames.length) {
		throw new IllegalStateException(String.format(
			"Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length));
	}

	for (int i = 0; i < fieldNames.length; i++) {
		final String name = fieldNames[i];

		final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i));
		reuse.set(name, fieldConverted);
	}

	return reuse;
}
 

private Row convertRow(JsonNode node, RowTypeInfo info) {
	final String[] names = info.getFieldNames();
	final TypeInformation<?>[] types = info.getFieldTypes();

	final Row row = new Row(names.length);
	for (int i = 0; i < names.length; i++) {
		final String name = names[i];
		final JsonNode subNode = node.get(name);
		if (subNode == null) {
			if (failOnMissingField) {
				throw new IllegalStateException(
					"Could not find field with name '" + name + "'.");
			} else {
				row.setField(i, null);
			}
		} else {
			row.setField(i, convert(subNode, types[i]));
		}
	}

	return row;
}
 

private HBaseTableSchema validateTableSchema(TableSchema schema) {
	HBaseTableSchema hbaseSchema = new HBaseTableSchema();
	String[] fieldNames = schema.getFieldNames();
	TypeInformation[] fieldTypes = schema.getFieldTypes();
	for (int i = 0; i < fieldNames.length; i++) {
		String name = fieldNames[i];
		TypeInformation<?> type = fieldTypes[i];
		if (type instanceof RowTypeInfo) {
			RowTypeInfo familyType = (RowTypeInfo) type;
			String[] qualifierNames = familyType.getFieldNames();
			TypeInformation[] qualifierTypes = familyType.getFieldTypes();
			for (int j = 0; j < familyType.getArity(); j++) {
				hbaseSchema.addColumn(name, qualifierNames[j], qualifierTypes[j].getTypeClass());
			}
		} else {
			hbaseSchema.setRowKey(name, type.getTypeClass());
		}
	}
	return hbaseSchema;
}
 

private static RowTypeInfo createReturnType(TableSchema leftTable, RowTypeInfo sideType) {
    String[] leftFields = leftTable.getColumnNames();
    TypeInformation[] leftTypes = leftTable.getTypes();
    int leftArity = leftFields.length;
    int rightArity = sideType.getArity();
    int size = leftArity + rightArity;
    String[] columnNames = new String[size];
    TypeInformation[] columnTypes = new TypeInformation[size];
    for (int i = 0; i < leftArity; i++) {
        columnNames[i] = leftFields[i];
        columnTypes[i] = leftTypes[i];
    }
    for (int i = 0; i < rightArity; i++) {
        columnNames[leftArity + i] = sideType.getFieldNames()[i];
        columnTypes[leftArity + i] = sideType.getTypeAt(i);
    }

    return new RowTypeInfo(columnTypes, columnNames);
}
 

private static RuntimeConverter createRowRuntimeConverter(
		RowTypeInfo rowTypeInfo,
		boolean ignoreParseErrors,
		boolean isTopLevel) {
	final TypeInformation<?>[] fieldTypes = rowTypeInfo.getFieldTypes();
	final String[] fieldNames = rowTypeInfo.getFieldNames();

	final RuntimeConverter[] fieldConverters =
		createFieldRuntimeConverters(ignoreParseErrors, fieldTypes);

	return assembleRowRuntimeConverter(ignoreParseErrors, isTopLevel, fieldNames, fieldConverters);
}
 

private static DataType convertToRowType(RowTypeInfo rowTypeInfo) {
	final String[] fieldNames = rowTypeInfo.getFieldNames();
	final DataTypes.Field[] fields = IntStream.range(0, rowTypeInfo.getArity())
		.mapToObj(i -> {
			DataType fieldType = toDataType(rowTypeInfo.getTypeAt(i));

			return DataTypes.FIELD(
				fieldNames[i],
				fieldType);
		})
		.toArray(DataTypes.Field[]::new);

	return DataTypes.ROW(fields).bridgedTo(Row.class);
}
 

/**
 * Convert {@link RowTypeInfo} to {@link CsvSchema}.
 */
public static CsvSchema convert(RowTypeInfo rowType) {
	final Builder builder = new CsvSchema.Builder();
	final String[] fields = rowType.getFieldNames();
	final TypeInformation<?>[] types = rowType.getFieldTypes();
	for (int i = 0; i < rowType.getArity(); i++) {
		builder.addColumn(new Column(i, fields[i], convertType(fields[i], types[i])));
	}
	return builder.build();
}
 

private ParquetTableSource(String path, MessageType parquetSchema, Configuration configuration,
								boolean recursiveEnumeration, @Nullable int[] selectedFields, @Nullable FilterPredicate predicate) {
	Preconditions.checkNotNull(path, "Path must not be null.");
	Preconditions.checkNotNull(parquetSchema, "ParquetSchema must not be null.");
	Preconditions.checkNotNull(configuration, "Configuration must not be null");
	this.path = path;
	this.parquetSchema = parquetSchema;
	this.parquetConfig = configuration;
	this.selectedFields = selectedFields;
	this.predicate = predicate;
	this.recursiveEnumeration = recursiveEnumeration;

	if (predicate != null) {
		this.isFilterPushedDown = true;
	}
	// determine the type information from the Parquet schema
	RowTypeInfo typeInfoFromSchema = (RowTypeInfo) ParquetSchemaConverter.fromParquetType(parquetSchema);

	// set return type info
	if (selectedFields == null) {
		this.typeInfo = typeInfoFromSchema;
	} else {
		this.typeInfo = RowTypeInfo.projectFields(typeInfoFromSchema, selectedFields);
	}

	// create a TableSchema that corresponds to the Parquet schema
	this.tableSchema = new TableSchema(
		typeInfoFromSchema.getFieldNames(),
		typeInfoFromSchema.getFieldTypes()
	);
}
 

private ParquetTableSource(String path, MessageType parquetSchema, Configuration configuration,
								boolean recursiveEnumeration, @Nullable int[] selectedFields, @Nullable FilterPredicate predicate) {
	Preconditions.checkNotNull(path, "Path must not be null.");
	Preconditions.checkNotNull(parquetSchema, "ParquetSchema must not be null.");
	Preconditions.checkNotNull(configuration, "Configuration must not be null");
	this.path = path;
	this.parquetSchema = parquetSchema;
	this.parquetConfig = configuration;
	this.selectedFields = selectedFields;
	this.predicate = predicate;
	this.recursiveEnumeration = recursiveEnumeration;

	if (predicate != null) {
		this.isFilterPushedDown = true;
	}
	// determine the type information from the Parquet schema
	RowTypeInfo typeInfoFromSchema = (RowTypeInfo) ParquetSchemaConverter.fromParquetType(parquetSchema);

	// set return type info
	if (selectedFields == null) {
		this.typeInfo = typeInfoFromSchema;
	} else {
		this.typeInfo = RowTypeInfo.projectFields(typeInfoFromSchema, selectedFields);
	}

	// create a TableSchema that corresponds to the Parquet schema
	this.tableSchema = new TableSchema(
		typeInfoFromSchema.getFieldNames(),
		typeInfoFromSchema.getFieldTypes()
	);
}
 

/**
 * Read parquet files with given parquet file schema.
 *
 * @param path The path of the file to read.
 * @param messageType schema of parquet file
 */

protected ParquetInputFormat(Path path, MessageType messageType) {
	super(path);
	this.expectedFileSchema = checkNotNull(messageType, "messageType");
	RowTypeInfo rowTypeInfo = (RowTypeInfo) ParquetSchemaConverter.fromParquetType(expectedFileSchema);
	this.fieldTypes = rowTypeInfo.getFieldTypes();
	this.fieldNames = rowTypeInfo.getFieldNames();
	// read whole parquet file as one file split
	this.unsplittable = true;
}
 

private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) {
	final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes();
	final String[] fieldNames = rowTypeInfo.getFieldNames();

	final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes);

	return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters);
}
 

private OrcTableSource(String path, TypeDescription orcSchema, Configuration orcConfig,
						int batchSize, boolean recursiveEnumeration,
						int[] selectedFields, Predicate[] predicates) {

	Preconditions.checkNotNull(path, "Path must not be null.");
	Preconditions.checkNotNull(orcSchema, "OrcSchema must not be null.");
	Preconditions.checkNotNull(path, "Configuration must not be null.");
	Preconditions.checkArgument(batchSize > 0, "Batch size must be larger than null.");
	this.path = path;
	this.orcSchema = orcSchema;
	this.orcConfig = orcConfig;
	this.batchSize = batchSize;
	this.recursiveEnumeration = recursiveEnumeration;
	this.selectedFields = selectedFields;
	this.predicates = predicates;

	// determine the type information from the ORC schema
	RowTypeInfo typeInfoFromSchema = (RowTypeInfo) OrcBatchReader.schemaToTypeInfo(this.orcSchema);

	// set return type info
	if (selectedFields == null) {
		this.typeInfo = typeInfoFromSchema;
	} else {
		this.typeInfo = RowTypeInfo.projectFields(typeInfoFromSchema, selectedFields);
	}

	// create a TableSchema that corresponds to the ORC schema
	this.tableSchema = new TableSchema(
		typeInfoFromSchema.getFieldNames(),
		typeInfoFromSchema.getFieldTypes()
	);
}
 

/**
 * Convert {@link RowTypeInfo} to {@link CsvSchema}.
 */
public static CsvSchema convert(RowTypeInfo rowType) {
	final Builder builder = new CsvSchema.Builder();
	final String[] fields = rowType.getFieldNames();
	final TypeInformation<?>[] types = rowType.getFieldTypes();
	for (int i = 0; i < rowType.getArity(); i++) {
		builder.addColumn(new Column(i, fields[i], convertType(fields[i], types[i])));
	}
	return builder.build();
}
 

private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) {
	final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes();
	final String[] fieldNames = rowTypeInfo.getFieldNames();

	final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes);

	return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters);
}
 

private OrcTableSource(String path, TypeDescription orcSchema, Configuration orcConfig,
						int batchSize, boolean recursiveEnumeration,
						int[] selectedFields, Predicate[] predicates) {

	Preconditions.checkNotNull(path, "Path must not be null.");
	Preconditions.checkNotNull(orcSchema, "OrcSchema must not be null.");
	Preconditions.checkNotNull(path, "Configuration must not be null.");
	Preconditions.checkArgument(batchSize > 0, "Batch size must be larger than null.");
	this.path = path;
	this.orcSchema = orcSchema;
	this.orcConfig = orcConfig;
	this.batchSize = batchSize;
	this.recursiveEnumeration = recursiveEnumeration;
	this.selectedFields = selectedFields;
	this.predicates = predicates;

	// determine the type information from the ORC schema
	RowTypeInfo typeInfoFromSchema = (RowTypeInfo) OrcBatchReader.schemaToTypeInfo(this.orcSchema);

	// set return type info
	if (selectedFields == null) {
		this.typeInfo = typeInfoFromSchema;
	} else {
		this.typeInfo = RowTypeInfo.projectFields(typeInfoFromSchema, selectedFields);
	}

	// create a TableSchema that corresponds to the ORC schema
	this.tableSchema = new TableSchema(
		typeInfoFromSchema.getFieldNames(),
		typeInfoFromSchema.getFieldTypes()
	);
}
 

private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) {
	final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes();
	final String[] fieldNames = rowTypeInfo.getFieldNames();

	final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes);

	return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters);
}
 

private static RuntimeConverter createRowRuntimeConverter(
		RowTypeInfo rowTypeInfo,
		boolean ignoreParseErrors,
		boolean isTopLevel) {
	final TypeInformation<?>[] fieldTypes = rowTypeInfo.getFieldTypes();
	final String[] fieldNames = rowTypeInfo.getFieldNames();

	final RuntimeConverter[] fieldConverters =
		createFieldRuntimeConverters(ignoreParseErrors, fieldTypes);

	return assembleRowRuntimeConverter(ignoreParseErrors, isTopLevel, fieldNames, fieldConverters);
}
 

/**
 * Convert {@link RowTypeInfo} to {@link CsvSchema}.
 */
public static CsvSchema convert(RowTypeInfo rowType) {
	final Builder builder = new CsvSchema.Builder();
	final String[] fields = rowType.getFieldNames();
	final TypeInformation<?>[] types = rowType.getFieldTypes();
	for (int i = 0; i < rowType.getArity(); i++) {
		builder.addColumn(new Column(i, fields[i], convertType(fields[i], types[i])));
	}
	return builder.build();
}
 

MockTableSource(List<Row> data, RowTypeInfo type) {
  this.data = data;
  this.type = type;
  this.schema = new TableSchema(type.getFieldNames(), type.getFieldTypes());
}