Java Code Examples for org.apache.flink.table.api.TableSchema#getFieldTypes()

/**
 * Transform the TableSchema to a string. The format of the string is comma separated colName-colType pairs,
 * such as "f0 int,f1 bigint,f2 string".
 *
 * @param schema the TableSchema to transform.
 * @return a string.
 */
public static String schema2SchemaStr(TableSchema schema) {
    String[] colNames = schema.getFieldNames();
    TypeInformation[] colTypes = schema.getFieldTypes();

    StringBuilder sbd = new StringBuilder();
    for (int i = 0; i < colNames.length; i++) {
        if (i > 0) {
            sbd.append(",");
        }
        String typeName;
        if (colTypes[i].equals(BYTE_PRIMITIVE_ARRAY_TYPE_INFO)) {
            typeName = "VARBINARY";
        } else if (colTypes[i].equals(VectorTypes.VECTOR)) {
            typeName = "VEC_TYPES_VECTOR";
        } else if (colTypes[i].equals(VectorTypes.DENSE_VECTOR)) {
            typeName = "VEC_TYPES_DENSE_VECTOR";
        } else if (colTypes[i].equals(VectorTypes.SPARSE_VECTOR)) {
            typeName = "VEC_TYPES_SPARSE_VECTOR";
        } else {
            typeName = FlinkTypeConverter.getTypeString(colTypes[i]);
        }
        sbd.append(colNames[i]).append(" ").append(typeName);
    }
    return sbd.toString();
}
 

private HBaseTableSchema validateTableSchema(TableSchema schema) {
	HBaseTableSchema hbaseSchema = new HBaseTableSchema();
	String[] fieldNames = schema.getFieldNames();
	TypeInformation[] fieldTypes = schema.getFieldTypes();
	for (int i = 0; i < fieldNames.length; i++) {
		String name = fieldNames[i];
		TypeInformation<?> type = fieldTypes[i];
		if (type instanceof RowTypeInfo) {
			RowTypeInfo familyType = (RowTypeInfo) type;
			String[] qualifierNames = familyType.getFieldNames();
			TypeInformation[] qualifierTypes = familyType.getFieldTypes();
			for (int j = 0; j < familyType.getArity(); j++) {
				hbaseSchema.addColumn(name, qualifierNames[j], qualifierTypes[j].getTypeClass());
			}
		} else {
			hbaseSchema.setRowKey(name, type.getTypeClass());
		}
	}
	return hbaseSchema;
}
 

/**
 * Adds a table schema under the given key.
 */
public void putTableSchema(String key, TableSchema schema) {
	checkNotNull(key);
	checkNotNull(schema);

	final String[] fieldNames = schema.getFieldNames();
	final TypeInformation<?>[] fieldTypes = schema.getFieldTypes();

	final List<List<String>> values = new ArrayList<>();
	for (int i = 0; i < schema.getFieldCount(); i++) {
		values.add(Arrays.asList(fieldNames[i], TypeStringUtils.writeTypeInfo(fieldTypes[i])));
	}

	putIndexedFixedProperties(
		key,
		Arrays.asList(TABLE_SCHEMA_NAME, TABLE_SCHEMA_TYPE),
		values);
}
 

/**
 * Adds a table schema under the given key.
 */
public void putTableSchema(String key, TableSchema schema) {
	checkNotNull(key);
	checkNotNull(schema);

	final String[] fieldNames = schema.getFieldNames();
	final TypeInformation<?>[] fieldTypes = schema.getFieldTypes();

	final List<List<String>> values = new ArrayList<>();
	for (int i = 0; i < schema.getFieldCount(); i++) {
		values.add(Arrays.asList(fieldNames[i], TypeStringUtils.writeTypeInfo(fieldTypes[i])));
	}

	putIndexedFixedProperties(
		key,
		Arrays.asList(TABLE_SCHEMA_NAME, TABLE_SCHEMA_TYPE),
		values);
}
 

/**
 * Return the columns in the table whose types are numeric and are not included in the excludeCols.
 *
 * <p>If <code>excludeCols</code> is null, return all the numeric columns.
 *
 * @param tableSchema TableSchema.
 * @param excludeCols the columns who are not considered.
 * @return numeric columns.
 */
public static String[] getNumericCols(TableSchema tableSchema, String[] excludeCols) {
	ArrayList<String> numericCols = new ArrayList<>();
	List<String> excludeColsList = (null == excludeCols ? null : Arrays.asList(excludeCols));
	String[] inColNames = tableSchema.getFieldNames();
	TypeInformation<?>[] inColTypes = tableSchema.getFieldTypes();

	for (int i = 0; i < inColNames.length; i++) {
		if (isSupportedNumericType(inColTypes[i])) {
			if (null == excludeColsList || !excludeColsList.contains(inColNames[i])) {
				numericCols.add(inColNames[i]);
			}
		}
	}

	return numericCols.toArray(new String[0]);
}
 

@Override
public void createTable(String tableName, TableSchema schema, Params parameter)
        throws ClassNotFoundException, SQLException, InstantiationException, IllegalAccessException {
    StringBuilder sbd = new StringBuilder();
    sbd.append("create table ").append(tableName).append(" (");
    String[] colNames = schema.getFieldNames();
    TypeInformation<?>[] colTypes = schema.getFieldTypes();
    int n = colNames.length;
    for (int i = 0; i < n; i++) {
        String type = FlinkTypeConverter.getTypeString(colTypes[i]);
        sbd.append(colNames[i]).append(" ").append(type);
        if ("VARCHAR".equalsIgnoreCase(type)) {
            sbd.append("(").append(getMaxVarcharLength()).append(")");
        }
        if (i < n - 1) {
            sbd.append(",");
        }
    }
    sbd.append(")");
    String sql = sbd.toString();
    LOG.info("JdbcDB create table {}: {}", tableName, sql);
    execute(sql);
}
 

/**
 * Creates a result. Might start threads or opens sockets so every created result must be closed.
 */
public <T> DynamicResult<T> createResult(Environment env, TableSchema schema, ExecutionConfig config) {

	final RowTypeInfo outputType = new RowTypeInfo(schema.getFieldTypes(), schema.getFieldNames());

	if (env.getExecution().inStreamingMode()) {
		// determine gateway address (and port if possible)
		final InetAddress gatewayAddress = getGatewayAddress(env.getDeployment());
		final int gatewayPort = getGatewayPort(env.getDeployment());

		if (env.getExecution().isChangelogMode()) {
			return new ChangelogCollectStreamResult<>(outputType, config, gatewayAddress, gatewayPort);
		} else {
			return new MaterializedCollectStreamResult<>(
				outputType,
				config,
				gatewayAddress,
				gatewayPort,
				env.getExecution().getMaxTableResultRows());
		}

	} else {
		// Batch Execution
		if (!env.getExecution().isTableMode()) {
			throw new SqlExecutionException("Results of batch queries can only be served in table mode.");
		}
		return new MaterializedCollectBatchResult<>(outputType, config);
	}
}
 

/**
 * Todo: add the exception throw
 *
 * @param schema
 * @return
 */
public static String toSchemaJson(TableSchema schema) {
    TypeInformation<?>[] types = schema.getFieldTypes();
    int n = types.length;
    String[] typeStrs = new String[n];
    for (int i = 0; i < n; i++) {
        typeStrs[i] = FlinkTypeConverter.getTypeString(types[i]);
        if (typeStrs[i] == null) {
            typeStrs[i] = VectorTypes.getTypeName(types[i]);
        }
    }
    return gson.toJson(new String[][]{schema.getFieldNames(), typeStrs}, String[][].class);
}
 

/**
 * Find the types of the <code>targetCols</code>. If the targetCol not exist, throw an exception.
 *
 * @param tableSchema TableSchema.
 * @param targetCols  the targetCols to find.
 * @return the corresponding types.
 */
public static TypeInformation[] findColTypesWithAssert(TableSchema tableSchema, String[] targetCols) {
    if (targetCols == null) {
        return tableSchema.getFieldTypes();
    }
    TypeInformation[] types = new TypeInformation[targetCols.length];
    for (int i = 0; i < types.length; i++) {
        types[i] = findColTypeWithAssert(tableSchema, targetCols[i]);
    }
    return types;
}
 

/**
 * Find the types of the <code>targetCols</code>. If the targetCol not exist, throw an exception.
 *
 * @param tableSchema TableSchema.
 * @param targetCols  the targetCols to find.
 * @return the corresponding types.
 */
public static TypeInformation[] findColTypesWithAssertAndHint(TableSchema tableSchema, String[] targetCols) {
    if (targetCols == null) {
        return tableSchema.getFieldTypes();
    }
    TypeInformation[] types = new TypeInformation[targetCols.length];
    for (int i = 0; i < types.length; i++) {
        types[i] = findColTypeWithAssertAndHint(tableSchema, targetCols[i]);
    }
    return types;
}
 

@Test
public void testKvParser() throws Exception {
    String kvStr = "f1=1,f2=2.0,f3=false,f4=val,f5=2018-09-10,f6=14:22:20,f7=2018-09-10 14:22:20";
    String schemaStr = "f1 bigint, f2 double, f3 boolean, f4 string, f5 date, f6 time, f7 timestamp";

    TableSchema schema = CsvUtil.schemaStr2Schema(schemaStr);
    StringParsers.KvParser parser = new StringParsers.KvParser(schema.getFieldNames(), schema.getFieldTypes(), ",", "=");
    Tuple2<Boolean, Row> parsed = parser.parse(kvStr);
    Assert.assertTrue(parsed.f0);
    Assert.assertEquals(parsed.f1.getArity(), 7);
}
 

/**
 * Find the types of the <code>targetCols</code>. If the targetCol not exist, return null.
 *
 * @param tableSchema TableSchema.
 * @param targetCols  the targetCols to find.
 * @return the corresponding types.
 */
public static TypeInformation<?>[] findColTypes(TableSchema tableSchema, String[] targetCols) {
	if (targetCols == null) {
		return tableSchema.getFieldTypes();
	}
	TypeInformation<?>[] types = new TypeInformation[targetCols.length];
	for (int i = 0; i < types.length; i++) {
		types[i] = findColType(tableSchema, targetCols[i]);
	}
	return types;
}
 

@Override
public <T> T transform(TableSchema param) throws Exception {
    TableSchema tableSchema = createTableSchema();
    if (tableSchema == null) {
        tableSchema = param;
    }
    if (tableSchema == null) {
        throw new IllegalArgumentException("TableSchema must be not null");
    }
    TypeInformation[] fieldTypes = new TypeInformation[tableSchema.getFieldCount()];
    for (int i = 0; i < tableSchema.getFieldCount(); i++) {
        if (FlinkTypeFactory.isTimeIndicatorType(tableSchema.getFieldTypes()[i])) {
            fieldTypes[i] = Types.SQL_TIMESTAMP();
        }else{
            fieldTypes[i] = tableSchema.getFieldTypes()[i];
        }
    }
    TypeInformation typeInformation = new RowTypeInfo(fieldTypes, tableSchema.getFieldNames());
    SerializationSchema<Row> rowSerializationSchema = createSerializationSchema(typeInformation);
    return (T) newTableSink(
        new TableSchema(tableSchema.getFieldNames(), fieldTypes),
        this.topic,
        PropertiesUtil.fromYamlMap(this.getProperties()),
        Optional.empty(),
        rowSerializationSchema == null ? new JsonRowSerializationSchema(typeInformation) : rowSerializationSchema
    );
}
 

private void validateSetOperation(
		SetQueryOperationType operationType,
		QueryOperation left,
		QueryOperation right) {
	TableSchema leftSchema = left.getTableSchema();
	int leftFieldCount = leftSchema.getFieldCount();
	TableSchema rightSchema = right.getTableSchema();
	int rightFieldCount = rightSchema.getFieldCount();

	if (leftFieldCount != rightFieldCount) {
		throw new ValidationException(
			format(
				"The %s operation on two tables of different column sizes: %d and %d is not supported",
				operationType.toString().toLowerCase(),
				leftFieldCount,
				rightFieldCount));
	}

	TypeInformation<?>[] leftFieldTypes = leftSchema.getFieldTypes();
	TypeInformation<?>[] rightFieldTypes = rightSchema.getFieldTypes();
	boolean sameSchema = IntStream.range(0, leftFieldCount)
		.allMatch(idx -> leftFieldTypes[idx].equals(rightFieldTypes[idx]));

	if (!sameSchema) {
		throw new ValidationException(
			format(
				"The %s operation on two tables of different schemas: %s and %s is not supported.",
				operationType.toString().toLowerCase(),
				leftSchema,
				rightSchema));
	}
}
 

@Override
public TableSchema getTableSchema(String tableName) throws Exception {
    TableSchema schema = getTableSchemaWithPartitionColumns(tableName);

    // remove static partition columns
    List<String> staticPartCols = getPartitionCols(tableName);
    int numPartCols = staticPartCols.size();
    if (numPartCols > 0) {
        Set<String> partColsSet = new HashSet<>();
        partColsSet.addAll(staticPartCols);
        int n = 0;
        String[] fieldNames = new String[schema.getFieldNames().length - numPartCols];
        TypeInformation[] fieldTypes = new TypeInformation[fieldNames.length];
        TypeInformation[] allFieldTypes = schema.getFieldTypes();
        for (int i = 0; i < allFieldTypes.length; i++) {
            String fieldName = schema.getFieldNames()[i];
            if (partColsSet.contains(fieldName)) {
                continue;
            }
            fieldNames[n] = fieldName;
            fieldTypes[n] = allFieldTypes[i];
            n++;
        }
        return new TableSchema(fieldNames, fieldTypes);
    } else {
        return schema;
    }
}
 

@Override
public RichOutputFormat createFormat(String tableName, TableSchema schema) {
    TypeInformation[] types = schema.getFieldTypes();
    String[] colNames = schema.getFieldNames();
    int[] parameterTypes = new int[types.length];
    for (int i = 0; i < types.length; ++i) {
        parameterTypes[i] = JdbcTypeConverter.getIntegerSqlType(types[i]);
    }
    StringBuilder sbd = new StringBuilder();
    sbd.append("INSERT INTO ").append(tableName).append(" (").append(colNames[0]);
    for (int i = 1; i < colNames.length; i++) {
        sbd.append(",").append(colNames[i]);
    }
    sbd.append(") VALUES (?");
    for (int i = 1; i < colNames.length; i++) {
        sbd.append(",").append("?");
    }
    sbd.append(")");

    return JDBCOutputFormat.buildJDBCOutputFormat()
            .setUsername(userName)
            .setPassword(password)
            .setDBUrl(getDbUrl())
            .setQuery(sbd.toString())
            .setDrivername(getDriverName())
            .setBatchInterval(5000)
            .setSqlTypes(parameterTypes)
            .finish();
}
 

/**
 * A utility function to extract label type from model schema.
 */
public static TypeInformation extractLabelType(TableSchema modelSchema) {
    return modelSchema.getFieldTypes()[2];
}
 

private RowTypeInfo createDestTable(String dbName, String tblName, TableSchema tableSchema, int numPartCols) throws Exception {
	CatalogTable catalogTable = createCatalogTable(tableSchema, numPartCols);
	hiveCatalog.createTable(new ObjectPath(dbName, tblName), catalogTable, false);
	return new RowTypeInfo(tableSchema.getFieldTypes(), tableSchema.getFieldNames());
}
 

@Override
public TypeInformation<Row> getReturnType() {
	TableSchema tableSchema = catalogTable.getSchema();
	return new RowTypeInfo(tableSchema.getFieldTypes(), tableSchema.getFieldNames());
}
 

/**
 * Find the type of the <code>targetCol</code>. If the targetCol not exist, return null.
 *
 * @param tableSchema TableSchema
 * @param targetCol   the targetCol to find.
 * @return the corresponding type.
 */
public static TypeInformation<?> findColType(TableSchema tableSchema, String targetCol) {
	int index = findColIndex(tableSchema.getFieldNames(), targetCol);

	return index == -1 ? null : tableSchema.getFieldTypes()[index];
}