Java Code Examples for org.apache.flink.table.api.TableSchema#getFieldNames()

/**
 * Return the columns in the table whose types are numeric and are not included in the excludeCols.
 *
 * <p>If <code>excludeCols</code> is null, return all the numeric columns.
 *
 * @param tableSchema TableSchema.
 * @param excludeCols the columns who are not considered.
 * @return numeric columns.
 */
public static String[] getNumericCols(TableSchema tableSchema, String[] excludeCols) {
	ArrayList<String> numericCols = new ArrayList<>();
	List<String> excludeColsList = (null == excludeCols ? null : Arrays.asList(excludeCols));
	String[] inColNames = tableSchema.getFieldNames();
	TypeInformation<?>[] inColTypes = tableSchema.getFieldTypes();

	for (int i = 0; i < inColNames.length; i++) {
		if (isSupportedNumericType(inColTypes[i])) {
			if (null == excludeColsList || !excludeColsList.contains(inColNames[i])) {
				numericCols.add(inColNames[i]);
			}
		}
	}

	return numericCols.toArray(new String[0]);
}
 

public void putSchema(ObjectPath tablePath, CatalogBaseTable table) throws IncompatibleSchemaException {
    String topic = objectPath2TopicName(tablePath);
    TableSchema tableSchema = table.getSchema();
    List<String> fieldsRemaining = new ArrayList<>(tableSchema.getFieldCount());
    for (String fieldName : tableSchema.getFieldNames()) {
        if (!PulsarOptions.META_FIELD_NAMES.contains(fieldName)) {
            fieldsRemaining.add(fieldName);
        }
    }

    DataType dataType;

    if (fieldsRemaining.size() == 1) {
        dataType = tableSchema.getFieldDataType(fieldsRemaining.get(0)).get();
    } else {
        List<DataTypes.Field> fieldList = fieldsRemaining.stream()
                .map(f -> DataTypes.FIELD(f, tableSchema.getFieldDataType(f).get()))
                .collect(Collectors.toList());
        dataType = DataTypes.ROW(fieldList.toArray(new DataTypes.Field[0]));
    }

    SchemaInfo si = SchemaUtils.sqlType2PulsarSchema(dataType).getSchemaInfo();
    SchemaUtils.uploadPulsarSchema(admin, topic, si);
}
 

private HBaseTableSchema validateTableSchema(TableSchema schema) {
	HBaseTableSchema hbaseSchema = new HBaseTableSchema();
	String[] fieldNames = schema.getFieldNames();
	TypeInformation[] fieldTypes = schema.getFieldTypes();
	for (int i = 0; i < fieldNames.length; i++) {
		String name = fieldNames[i];
		TypeInformation<?> type = fieldTypes[i];
		if (type instanceof RowTypeInfo) {
			RowTypeInfo familyType = (RowTypeInfo) type;
			String[] qualifierNames = familyType.getFieldNames();
			TypeInformation[] qualifierTypes = familyType.getFieldTypes();
			for (int j = 0; j < familyType.getArity(); j++) {
				hbaseSchema.addColumn(name, qualifierNames[j], qualifierTypes[j].getTypeClass());
			}
		} else {
			hbaseSchema.setRowKey(name, type.getTypeClass());
		}
	}
	return hbaseSchema;
}
 

@Test
public void testJsonParser() throws Exception {
    String jsonStr = "{\n" +
        "  \"media_name\": \"Titanic\",\n" +
        "  \"title\": \"Titanic\",\n" +
        "  \"compare_point\": 0.0001,\n" +
        "  \"spider_point\": 0.0000,\n" +
        "  \"search_point\": 0.6,\n" +
        "  \"collection_id\": 123456,\n" +
        "  \"media_id\": 3214\n" +
        "}";

    String schemaStr
        = "media_name string, title string, compare_point double, spider_point double, search_point double, "
        + "collection_id bigint, media_id bigint";

    TableSchema schema = CsvUtil.schemaStr2Schema(schemaStr);
    StringParsers.JsonParser parser = new StringParsers.JsonParser(schema.getFieldNames(), schema.getFieldTypes());
    Tuple2<Boolean, Row> parsed = parser.parse(jsonStr);
    Assert.assertTrue(parsed.f0);
    Assert.assertEquals(parsed.f1.getArity(), 7);
}
 

public HiveTableOutputFormat(JobConf jobConf, ObjectPath tablePath, CatalogTable table, HiveTablePartition hiveTablePartition,
							Properties tableProperties, boolean overwrite) {
	super(jobConf.getCredentials());

	Preconditions.checkNotNull(table, "table cannot be null");
	Preconditions.checkNotNull(hiveTablePartition, "HiveTablePartition cannot be null");
	Preconditions.checkNotNull(tableProperties, "Table properties cannot be null");

	HadoopUtils.mergeHadoopConf(jobConf);
	this.jobConf = jobConf;
	this.tablePath = tablePath;
	this.partitionColumns = table.getPartitionKeys();
	TableSchema tableSchema = table.getSchema();
	this.fieldNames = tableSchema.getFieldNames();
	this.fieldTypes = tableSchema.getFieldDataTypes();
	this.hiveTablePartition = hiveTablePartition;
	this.tableProperties = tableProperties;
	this.overwrite = overwrite;
	isPartitioned = partitionColumns != null && !partitionColumns.isEmpty();
	isDynamicPartition = isPartitioned && partitionColumns.size() > hiveTablePartition.getPartitionSpec().size();
	hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
}
 

public NumericalTypeCastMapper(TableSchema dataSchema, Params params) {
	super(dataSchema, params);
	String[] inputColNames = this.params.get(NumericalTypeCastParams.SELECTED_COLS);
	this.colIndices = TableUtil.findColIndicesWithAssertAndHint(dataSchema.getFieldNames(), inputColNames);
	String[] outputColNames = params.get(NumericalTypeCastParams.OUTPUT_COLS);
	if (outputColNames == null || outputColNames.length == 0) {
		outputColNames = inputColNames;
	}

	String[] reservedColNames = params.get(NumericalTypeCastParams.RESERVED_COLS);

	if (reservedColNames == null || reservedColNames.length == 0) {
		reservedColNames = dataSchema.getFieldNames();
	}

	targetType = FlinkTypeConverter.getFlinkType(params.get(HasTargetType.TARGET_TYPE).toString());

	this.outputColsHelper = new OutputColsHelper(dataSchema, outputColNames,
		Arrays.stream(outputColNames).map(x -> targetType).toArray(TypeInformation[]::new), reservedColNames);
}
 

@Override
public void loadModel(List <Row> modelRows) {
	LinearModelDataConverter softmaxConverter
		= new LinearModelDataConverter();
	model = softmaxConverter.load(modelRows);
	TableSchema dataSchema = getDataSchema();
	if (vectorColIndex == -1) {
		if (this.model.featureNames != null) {
			this.featureN = this.model.featureNames.length;
			this.featureIdx = new int[this.featureN];
			String[] predictTableColNames = dataSchema.getFieldNames();
			for (int i = 0; i < this.featureN; i++) {
				this.featureIdx[i] = TableUtil.findColIndexWithAssert(predictTableColNames,
					this.model.featureNames[i]);
			}
		} else {
			vectorColIndex = TableUtil.findColIndexWithAssert(dataSchema.getFieldNames(), model.vectorColName);
		}
	}
}
 

/**
 * Load model from the row type data.
 *
 * @param modelRows the list of Row type data
 */
@Override
public void loadModel(List<Row> modelRows) {
    this.model = new LinearModelDataConverter(LinearModelDataConverter.extractLabelType(super.getModelSchema()))
            .load(modelRows);
    if (vectorColIndex == -1) {
        TableSchema dataSchema = getDataSchema();
        if (this.model.featureNames != null) {
            this.featureN = this.model.featureNames.length;
            this.featureIdx = new int[this.featureN];
            String[] predictTableColNames = dataSchema.getFieldNames();
            for (int i = 0; i < this.featureN; i++) {
                this.featureIdx[i] = TableUtil.findColIndexWithAssert(predictTableColNames,
                        this.model.featureNames[i]);
            }
        } else {
            vectorColIndex = TableUtil.findColIndexWithAssert(dataSchema.getFieldNames(), model.vectorColName);
        }
    }
}
 

/**
 * Return the columns in the table whose types are numeric and are not included in the excludeCols.
 * <p>
 * <p>If <code>excludeCols</code> is null, return all the numeric columns.
 *
 * @param tableSchema TableSchema.
 * @param excludeCols the columns who are not considered.
 * @return numeric columns.
 */
public static String[] getNumericCols(TableSchema tableSchema, String[] excludeCols) {
    ArrayList<String> numericCols = new ArrayList<>();
    List<String> excludeColsList = (null == excludeCols ? null : Arrays.asList(excludeCols));
    String[] inColNames = tableSchema.getFieldNames();
    TypeInformation<?>[] inColTypes = tableSchema.getFieldTypes();

    for (int i = 0; i < inColNames.length; i++) {
        if (isNumber(inColTypes[i])) {
            if (null == excludeColsList || !excludeColsList.contains(inColNames[i])) {
                numericCols.add(inColNames[i]);
            }
        }
    }

    return numericCols.toArray(new String[0]);
}
 

private JdbcTableSource(
	JdbcOptions options, JdbcReadOptions readOptions, JdbcLookupOptions lookupOptions,
	TableSchema schema, int[] selectFields) {
	this.options = options;
	this.readOptions = readOptions;
	this.lookupOptions = lookupOptions;
	this.schema = schema;

	this.selectFields = selectFields;

	final DataType[] schemaDataTypes = schema.getFieldDataTypes();
	final String[] schemaFieldNames = schema.getFieldNames();
	if (selectFields != null) {
		DataType[] dataTypes = new DataType[selectFields.length];
		String[] fieldNames = new String[selectFields.length];
		for (int i = 0; i < selectFields.length; i++) {
			dataTypes[i] = schemaDataTypes[selectFields[i]];
			fieldNames[i] = schemaFieldNames[selectFields[i]];
		}
		this.producedDataType =
				TableSchema.builder().fields(fieldNames, dataTypes).build().toRowDataType();
	} else {
		this.producedDataType = schema.toRowDataType();
	}
}
 

/**
 * Extract select col types from model schema.
 */
public static TypeInformation[] extractSelectedColTypes(TableSchema modelSchema) {
    int selectedColNum = modelSchema.getFieldNames().length - 2;
    TypeInformation[] selectedColTypes = new TypeInformation[selectedColNum];
    for (int i = 0; i < selectedColNum; i++) {
        selectedColTypes[i] = modelSchema.getFieldType(2 + i).get();
    }
    return selectedColTypes;
}
 

private HBaseTableSchema validateTableSchema(TableSchema schema) {
	HBaseTableSchema hbaseSchema = new HBaseTableSchema();
	String[] fieldNames = schema.getFieldNames();
	TypeInformation[] fieldTypes = schema.getFieldTypes();
	for (int i = 0; i < fieldNames.length; i++) {
		String name = fieldNames[i];
		TypeInformation<?> type = fieldTypes[i];
		if (type instanceof RowTypeInfo) {
			RowTypeInfo familyType = (RowTypeInfo) type;
			String[] qualifierNames = familyType.getFieldNames();
			TypeInformation[] qualifierTypes = familyType.getFieldTypes();
			for (int j = 0; j < familyType.getArity(); j++) {
				// HBase connector doesn't support LocalDateTime
				// use Timestamp as conversion class for now.
				Class clazz = qualifierTypes[j].getTypeClass();
				if (LocalDateTime.class.equals(clazz)) {
					clazz = Timestamp.class;
				} else if (LocalDate.class.equals(clazz)) {
					clazz = Date.class;
				} else if (LocalTime.class.equals(clazz)) {
					clazz = Time.class;
				}
				hbaseSchema.addColumn(name, qualifierNames[j], clazz);
			}
		} else {
			hbaseSchema.setRowKey(name, type.getTypeClass());
		}
	}
	return hbaseSchema;
}
 

private TableSchema getProducedTableSchema() {
	TableSchema fullSchema = getTableSchema();
	if (projectedFields == null) {
		return fullSchema;
	} else {
		String[] fullNames = fullSchema.getFieldNames();
		DataType[] fullTypes = fullSchema.getFieldDataTypes();
		return TableSchema.builder().fields(
				Arrays.stream(projectedFields).mapToObj(i -> fullNames[i]).toArray(String[]::new),
				Arrays.stream(projectedFields).mapToObj(i -> fullTypes[i]).toArray(DataType[]::new)).build();
	}
}
 

public CsvWriter(TableSchema schema, String fieldDelim, Character quoteChar) {

		this.colNames = schema.getFieldNames();
		this.fieldDelim = fieldDelim;
		this.enableQuote = quoteChar != null;
		if (enableQuote) {
			this.quoteString = quoteChar.toString();
			this.escapedQuote = this.quoteString + this.quoteString;
		} else {
			this.quoteString = null;
			this.escapedQuote = null;
		}
	}
 

private boolean checkForCustomFieldMapping(DescriptorProperties descriptorProperties, TableSchema schema) {
	final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping(
		descriptorProperties,
		Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type
	return fieldMapping.size() != schema.getFieldNames().length ||
		!fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue()));
}
 

/**
 * Extract select col names from model schema.
 */
public static String[] extractSelectedColNames(TableSchema modelSchema) {
    int selectedColNum = modelSchema.getFieldNames().length - 2;
    String[] selectedColNames = new String[selectedColNum];
    for (int i = 0; i < selectedColNum; i++) {
        selectedColNames[i] = modelSchema.getFieldNames()[2 + i];
    }
    return selectedColNames;
}
 

/**
 * Create Hive columns from Flink TableSchema.
 */
public static List<FieldSchema> createHiveColumns(TableSchema schema) {
	String[] fieldNames = schema.getFieldNames();
	DataType[] fieldTypes = schema.getFieldDataTypes();

	List<FieldSchema> columns = new ArrayList<>(fieldNames.length);

	for (int i = 0; i < fieldNames.length; i++) {
		columns.add(
			new FieldSchema(fieldNames[i], HiveTypeUtil.toHiveTypeName(fieldTypes[i]), null));
	}

	return columns;
}
 

/**
 * Creates the sink function based on the given table schema and current builder state.
 * @param tableSchema the schema of the sink table
 */
public FlinkPravegaWriter<Row> createSinkFunction(TableSchema tableSchema) {
    Preconditions.checkState(routingKeyFieldName != null, "The routing key field must be provided.");
    Preconditions.checkState(serializationSchema != null, "The serializationSchema must be provided.");
    PravegaEventRouter<Row> eventRouter = new FlinkPravegaTableSink.RowBasedRouter(routingKeyFieldName, tableSchema.getFieldNames(), tableSchema.getFieldDataTypes());
    return createSinkFunction(serializationSchema, eventRouter);
}
 

/** Convert CREATE VIEW statement. */
private Operation convertCreateView(SqlCreateView sqlCreateView) {
	final SqlNode query = sqlCreateView.getQuery();
	final SqlNodeList fieldList = sqlCreateView.getFieldList();

	SqlNode validateQuery = flinkPlanner.validate(query);
	PlannerQueryOperation operation = toQueryOperation(flinkPlanner, validateQuery);
	TableSchema schema = operation.getTableSchema();

	// the view column list in CREATE VIEW is optional, if it's not empty, we should update
	// the column name with the names in view column list.
	if (!fieldList.getList().isEmpty()) {
		// alias column names
		String[] inputFieldNames = schema.getFieldNames();
		String[] aliasFieldNames = fieldList.getList().stream()
				.map(SqlNode::toString)
				.toArray(String[]::new);

		if (inputFieldNames.length != aliasFieldNames.length) {
			throw new SqlConversionException(String.format(
					"VIEW definition and input fields not match:\n\tDef fields: %s.\n\tInput fields: %s.",
					Arrays.toString(aliasFieldNames), Arrays.toString(inputFieldNames)));
		}

		DataType[] inputFieldTypes = schema.getFieldDataTypes();
		schema = TableSchema.builder().fields(aliasFieldNames, inputFieldTypes).build();
	}

	String originalQuery = getQuotedSqlString(query);
	String expandedQuery = getQuotedSqlString(validateQuery);
	String comment = sqlCreateView.getComment().map(c -> c.getNlsString().getValue()).orElse(null);
	CatalogView catalogView = new CatalogViewImpl(originalQuery,
			expandedQuery,
			schema,
			Collections.emptyMap(),
			comment);

	UnresolvedIdentifier unresolvedIdentifier = UnresolvedIdentifier.of(sqlCreateView.fullViewName());
	ObjectIdentifier identifier = catalogManager.qualifyIdentifier(unresolvedIdentifier);

	return new CreateViewOperation(
			identifier,
			catalogView,
			sqlCreateView.isIfNotExists(),
			sqlCreateView.isTemporary());
}
 

public ModelMapper(TableSchema modelSchema, TableSchema dataSchema, Params params) {
    super(dataSchema, params);
    this.modelFieldNames = modelSchema.getFieldNames();
    this.modelFieldTypes = modelSchema.getFieldDataTypes();
}