Java Code Examples for org.apache.flink.api.common.typeinfo.Types#DOUBLE
The following examples show how to use
org.apache.flink.api.common.typeinfo.Types#DOUBLE .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ImputerMapperTest.java From Alink with Apache License 2.0 | 6 votes |
|
@Test
public void testMean() throws Exception {
Row[] rows = new Row[]{
Row.of(0L, "{\"selectedCols\":\"[\\\"f_double\\\",\\\"f_long\\\",\\\"f_int\\\"]\",\"strategy\":\"\\\"mean\\\"\"}", null, null, null),
Row.of(1048576L, "[0.3333333333333333,1.0,1.0]", null, null, null)
};
List<Row> model = Arrays.asList(rows);
TableSchema dataSchema = new TableSchema(
new String[]{"f_string", "f_long", "f_int", "f_double", "f_boolean"},
new TypeInformation<?>[]{Types.STRING, Types.LONG, Types.INT, Types.DOUBLE, Types.BOOLEAN}
);
Params params = new Params();
ImputerModelMapper mapper = new ImputerModelMapper(modelSchema, dataSchema, params);
mapper.loadModel(model);
assertEquals(mapper.map(Row.of("a", null, null, null, true)).getField(1), 1L);
assertEquals(mapper.map(Row.of("a", null, null, null, true)).getField(2), 1);
assertEquals((double) mapper.map(Row.of("a", null, null, null, true)).getField(3), 0.333333333, 10e-4);
}
Example 2
| Source File: StandardScalerTest.java From Alink with Apache License 2.0 | 6 votes |
|
public static AlgoOperator getMultiTypeData(boolean isBatch) {
Row[] testArray =
new Row[]{
Row.of(new Object[]{"0", "a", 1L, 1, 0.2, true}),
Row.of(new Object[]{"1", null, 2L, 2, null, true}),
Row.of(new Object[]{"2", "c", null, null, null, false}),
Row.of(new Object[]{"3", "a", 0L, 0, null, null}),
};
String[] colNames = new String[]{"id", "f_string", "f_long", "f_int", "f_double", "f_boolean"};
TypeInformation[] colTypes = new TypeInformation[]{Types.STRING, Types.STRING, Types.LONG, Types.INT, Types.DOUBLE,
Types.BOOLEAN};
TableSchema schema = new TableSchema(
colNames,
colTypes
);
if (isBatch) {
return new MemSourceBatchOp(Arrays.asList(testArray), schema);
} else {
return new MemSourceStreamOp(Arrays.asList(testArray), schema);
}
}
Example 3
| Source File: ImputerTest.java From Alink with Apache License 2.0 | 6 votes |
|
AlgoOperator getData(boolean isBatch) {
Row[] testArray =
new Row[]{
Row.of("0", "a", 1L, 1, 2.0, true),
Row.of("1", null, 2L, 2, -3.0, true),
Row.of("2", "c", null, null, 2.0, false),
Row.of("3", "a", 0L, 0, null, null),
};
String[] colNames = new String[]{"id", "f_string", "f_long", "f_int", "f_double", "f_boolean"};
TableSchema schema = new TableSchema(
colNames,
new TypeInformation<?>[] {Types.STRING, Types.STRING, Types.LONG, Types.INT, Types.DOUBLE, Types.BOOLEAN}
);
if (isBatch) {
return new MemSourceBatchOp(Arrays.asList(testArray), schema);
} else {
return new MemSourceStreamOp(Arrays.asList(testArray), schema);
}
}
Example 4
| Source File: VectorAssemblerMapperTest.java From Alink with Apache License 2.0 | 6 votes |
|
@Test
public void testToSparse() throws Exception {
TableSchema schema = new TableSchema(new String[]{"c0", "c1", "c2"},
new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING});
TableSchema outSchema = new TableSchema(new String[]{"c0", "out"},
new TypeInformation<?>[]{Types.STRING, VectorTypes.VECTOR});
Params params = new Params()
.set(VectorAssemblerParams.SELECTED_COLS, new String[]{"c0", "c1", "c2"})
.set(VectorAssemblerParams.OUTPUT_COL, "out")
.set(VectorAssemblerParams.RESERVED_COLS, new String[]{"c0"});
VectorAssemblerMapper mapper = new VectorAssemblerMapper(schema, params);
/* only reverse one column. */
assertEquals(mapper.map(Row.of(new DenseVector(new double[]{3.0, 4.0}), 3.0, new SparseVector(11, new int[]{0, 10}, new double[]{1.0, 4.0}))).getField(1),
new SparseVector(14, new int[]{0, 1, 2, 3, 13}, new double[]{3.0, 4.0, 3.0, 1.0, 4.0}));
assertEquals(mapper.getOutputSchema(), outSchema);
}
Example 5
| Source File: KMeansOldModelMapper2Test.java From Alink with Apache License 2.0 | 6 votes |
|
@Test
public void testHaversineDistance() throws Exception {
Row[] rows = new Row[] {
Row.of(0L, "{\"vectorCol\":null,\"latitudeCol\":\"\\\"f1\\\"\",\"longitudeCol\":\"\\\"f0\\\"\","
+ "\"distanceType\":\"\\\"HAVERSINE\\\"\",\"k\":\"2\",\"modelSchema\":\"\\\"model_id bigint,"
+ "model_info string\\\"\",\"isNewFormat\":\"true\",\"vectorSize\":\"2\"}"),
Row.of(1048576L, "{\"center\":\"{\\\"data\\\":[8.33,9.0]}\",\"clusterId\":0,\"weight\":3.0}"),
Row.of(2097152L, "{\"center\":\"{\\\"data\\\":[1.0,1.33]}\",\"clusterId\":1,\"weight\":3.0}")
};
List<Row> model = Arrays.asList(rows);
TableSchema modelSchema = new KMeansModelDataConverter().getModelSchema();
TableSchema dataSchema = new TableSchema(
new String[] {"f0", "f1"}, new TypeInformation<?>[] {Types.DOUBLE, Types.DOUBLE}
);
Params params = new Params()
.set(KMeansPredictParams.PREDICTION_COL, "pred");
KMeansModelMapper mapper = new KMeansModelMapper(modelSchema, dataSchema, params);
mapper.loadModel(model);
assertEquals(mapper.map(Row.of(0, 0)).getField(2), 1L);
assertEquals(mapper.getOutputSchema(), new TableSchema(new String[] {"f0", "f1", "pred"},
new TypeInformation<?>[] {Types.DOUBLE, Types.DOUBLE, Types.LONG}));
}
Example 6
| Source File: VectorAssemblerMapperTest.java From Alink with Apache License 2.0 | 6 votes |
|
@Test
public void testSkip() throws Exception {
TableSchema schema = new TableSchema(new String[]{"c0", "c1", "c2"},
new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING});
TableSchema outSchema = new TableSchema(new String[]{"c0", "out"},
new TypeInformation<?>[]{Types.STRING, VectorTypes.VECTOR});
Params params = new Params()
.set(VectorAssemblerParams.SELECTED_COLS, new String[]{"c0", "c1", "c2"})
.set(VectorAssemblerParams.OUTPUT_COL, "out")
.set(VectorAssemblerParams.HANDLE_INVALID, HandleInvalidMethod.SKIP)
.set(VectorAssemblerParams.RESERVED_COLS, new String[]{"c0"});
VectorAssemblerMapper mapper = new VectorAssemblerMapper(schema, params);
/* skip the invalid data. */
assertEquals(mapper.map(Row.of(new DenseVector(new double[]{3.0, 4.0}), null, new SparseVector(11, new int[]{0, 10}, new double[]{1.0, 4.0}))).getField(1),
null);
assertEquals(mapper.getOutputSchema(), outSchema);
}
Example 7
| Source File: StandardScalerMapperTest.java From Alink with Apache License 2.0 | 6 votes |
|
@Test
public void test() throws Exception {
Row[] rows = new Row[]{
Row.of(0L, "{\"withMean\":\"true\",\"withStd\":\"true\"}", null, null, null),
Row.of(1048576L, "[1.0,1.0,0.2]", null, null, null),
Row.of(2097152L, "[1.0,1.0,1.0]", null, null, null)
};
List<Row> model = Arrays.asList(rows);
TableSchema dataSchema = new TableSchema(
new String[]{"f_string", "f_long", "f_int", "f_double", "f_boolean"},
new TypeInformation<?>[]{Types.STRING, Types.LONG, Types.INT, Types.DOUBLE, Types.BOOLEAN}
);
Params params = new Params();
StandardScalerModelMapper mapper = new StandardScalerModelMapper(modelSchema, dataSchema, params);
mapper.loadModel(model);
assertEquals((double) mapper.map(Row.of("a", 1L, 1, 2.0, true)).getField(1), 0.0, 10e-4);
assertEquals((double) mapper.map(Row.of("a", 1L, 1, 2.0, true)).getField(2), 0.8, 10e-4);
assertEquals((double) mapper.map(Row.of("a", 1L, 1, 2.0, true)).getField(3), 1.0, 10e-4);
}
Example 8
| Source File: LdaTrainBatchOp.java From Alink with Apache License 2.0 | 6 votes |
|
/**
* Save the word-topic model in the sideOutputs.
*/
private void saveWordTopicModelAndPerplexity(DataSet<Row> model, int numTopic,
Boolean ifOnline) {
DataSet<Row> wordTopicDataSet;
if (ifOnline) {
wordTopicDataSet = model.mapPartition(new BuildWordTopicModelOnline()).setParallelism(1);
} else {
wordTopicDataSet = model.mapPartition(new BuildWordTopicModelGibbs()).setParallelism(1);
}
String[] colNames = new String[numTopic + 1];
TypeInformation[] colTypes = new TypeInformation[colNames.length];
colNames[0] = "word";
colTypes[0] = Types.STRING;
for (int i = 0; i < numTopic; i++) {
colNames[1 + i] = "topic_" + i;
colTypes[1 + i] = Types.DOUBLE;
}
DataSet<Row> logPerplexity = model.mapPartition(new CalculatePerplexityAndLikelihood()).setParallelism(1);
this.setSideOutputTables(new Table[] {
DataSetConversionUtil.toTable(getMLEnvironmentId(), wordTopicDataSet, colNames, colTypes),
DataSetConversionUtil.toTable(getMLEnvironmentId(),
logPerplexity, new String[]{"logPerplexity", "logLikelihood"},
new TypeInformation[]{Types.DOUBLE, Types.DOUBLE})
});
}
Example 9
| Source File: JsonRowDeserializationSchema.java From flink with Apache License 2.0 | 5 votes |
|
private Optional<DeserializationRuntimeConverter> createConverterForSimpleType(TypeInformation<?> simpleTypeInfo) {
if (simpleTypeInfo == Types.VOID) {
return Optional.of((mapper, jsonNode) -> null);
} else if (simpleTypeInfo == Types.BOOLEAN) {
return Optional.of(this::convertToBoolean);
} else if (simpleTypeInfo == Types.STRING) {
return Optional.of((mapper, jsonNode) -> jsonNode.asText());
} else if (simpleTypeInfo == Types.INT) {
return Optional.of(this::convertToInt);
} else if (simpleTypeInfo == Types.LONG) {
return Optional.of(this::convertToLong);
} else if (simpleTypeInfo == Types.DOUBLE) {
return Optional.of(this::convertToDouble);
} else if (simpleTypeInfo == Types.FLOAT) {
return Optional.of((mapper, jsonNode) -> Float.parseFloat(jsonNode.asText().trim()));
} else if (simpleTypeInfo == Types.SHORT) {
return Optional.of((mapper, jsonNode) -> Short.parseShort(jsonNode.asText().trim()));
} else if (simpleTypeInfo == Types.BYTE) {
return Optional.of((mapper, jsonNode) -> Byte.parseByte(jsonNode.asText().trim()));
} else if (simpleTypeInfo == Types.BIG_DEC) {
return Optional.of(this::convertToBigDecimal);
} else if (simpleTypeInfo == Types.BIG_INT) {
return Optional.of(this::convertToBigInteger);
} else if (simpleTypeInfo == Types.SQL_DATE) {
return Optional.of(this::convertToDate);
} else if (simpleTypeInfo == Types.SQL_TIME) {
return Optional.of(this::convertToTime);
} else if (simpleTypeInfo == Types.SQL_TIMESTAMP) {
return Optional.of(this::convertToTimestamp);
} else if (simpleTypeInfo == Types.LOCAL_DATE) {
return Optional.of(this::convertToLocalDate);
} else if (simpleTypeInfo == Types.LOCAL_TIME) {
return Optional.of(this::convertToLocalTime);
} else if (simpleTypeInfo == Types.LOCAL_DATE_TIME) {
return Optional.of(this::convertToLocalDateTime);
} else {
return Optional.empty();
}
}
Example 10
| Source File: WordCountUtil.java From Alink with Apache License 2.0 | 5 votes |
|
private static BatchOperator trans(BatchOperator in, String[] selectedColNames, String[] keepColNames,
BatchOperator indexedVocab, String wordColName, String idxColName,
boolean isWord, String wordDelimiter) {
String[] colnames = in.getColNames();
TypeInformation <?>[] coltypes = in.getColTypes();
int[] colIdxs = findColIdx(selectedColNames, colnames, coltypes);
int[] appendIdxs = findAppendColIdx(keepColNames, colnames);
// only 2 cols: word, idx
DataSet <Row> voc = indexedVocab.select(wordColName + "," + idxColName).getDataSet();
DataSet <Row> contentMapping = in.getDataSet()
.map(new GenContentMapping(colIdxs, appendIdxs, isWord, wordDelimiter))
.withBroadcastSet(voc, "vocabulary");
int transColSize = colIdxs.length;
int keepColSize = keepColNames == null ? 0 : keepColNames.length;
int outputColSize = transColSize + keepColSize;
String[] names = new String[outputColSize];
TypeInformation <?>[] types = new TypeInformation <?>[outputColSize];
int i = 0;
for (; i < transColSize; ++i) {
names[i] = colnames[colIdxs[i]];
types[i] = isWord ? Types.DOUBLE : Types.STRING;
}
for (; i < outputColSize; ++i) {
names[i] = colnames[appendIdxs[i - transColSize]];
types[i] = coltypes[appendIdxs[i - transColSize]];
}
return new TableSourceBatchOp(DataSetConversionUtil.toTable(in.getMLEnvironmentId(), contentMapping, names, types))
.setMLEnvironmentId(in.getMLEnvironmentId());
}
Example 11
| Source File: SavepointWriterITCase.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
ValueStateDescriptor<Double> descriptor = new ValueStateDescriptor<>("total", Types.DOUBLE);
state = getRuntimeContext().getState(descriptor);
}
Example 12
| Source File: OrcTableSourceTest.java From flink with Apache License 2.0 | 5 votes |
|
private TypeInformation[] getNestedFieldTypes() {
return new TypeInformation[]{
Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, Types.STRING,
Types.ROW_NAMED(
new String[]{"list"},
ObjectArrayTypeInfo.getInfoFor(
Types.ROW_NAMED(
new String[]{"int1", "string1"},
Types.INT, Types.STRING
)
)
),
ObjectArrayTypeInfo.getInfoFor(
Types.ROW_NAMED(
new String[]{"int1", "string1"},
Types.INT, Types.STRING
)
),
new MapTypeInfo<>(
Types.STRING,
Types.ROW_NAMED(
new String[]{"int1", "string1"},
Types.INT, Types.STRING
)
)
};
}
Example 13
| Source File: OrcTableSourceTest.java From flink with Apache License 2.0 | 5 votes |
|
private TypeInformation[] getNestedFieldTypes() {
return new TypeInformation[]{
Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, Types.STRING,
Types.ROW_NAMED(
new String[]{"list"},
ObjectArrayTypeInfo.getInfoFor(
Types.ROW_NAMED(
new String[]{"int1", "string1"},
Types.INT, Types.STRING
)
)
),
ObjectArrayTypeInfo.getInfoFor(
Types.ROW_NAMED(
new String[]{"int1", "string1"},
Types.INT, Types.STRING
)
),
new MapTypeInfo<>(
Types.STRING,
Types.ROW_NAMED(
new String[]{"int1", "string1"},
Types.INT, Types.STRING
)
)
};
}
Example 14
| Source File: ApproxVectorSimilarityJoinLSHBatchOp.java From Alink with Apache License 2.0 | 5 votes |
|
static TableSchema getJoinOutputSchema(BatchOperator[] inputs, String leftIdCol, String rightIdCol) {
TypeInformation[] types = new TypeInformation[] {
TableUtil.findColTypeWithAssertAndHint(inputs[0].getSchema(), leftIdCol),
TableUtil.findColTypeWithAssertAndHint(inputs[1].getSchema(), rightIdCol),
Types.DOUBLE};
if (leftIdCol.equalsIgnoreCase(rightIdCol)) {
leftIdCol = leftIdCol + "_left";
rightIdCol = rightIdCol + "_right";
}
String[] names = new String[] {leftIdCol, rightIdCol, DISTANCE_COL};
return new TableSchema(names, types);
}
Example 15
| Source File: FirstValueAggFunction.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public TypeInformation<Double> getResultType() {
return Types.DOUBLE;
}
Example 16
| Source File: FirstValueWithRetractAggFunction.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public TypeInformation<Double> getResultType() {
return Types.DOUBLE;
}
Example 17
| Source File: LastValueAggFunction.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public TypeInformation<Double> getResultType() {
return Types.DOUBLE;
}
Example 18
| Source File: SavepointWriterITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void open(Configuration parameters) {
ValueStateDescriptor<Double> descriptor = new ValueStateDescriptor<>("total", Types.DOUBLE);
state = getRuntimeContext().getState(descriptor);
}
Example 19
| Source File: OneVsRest.java From Alink with Apache License 2.0 | 4 votes |
|
private static BatchOperator generateTrainData(BatchOperator data, BatchOperator allLabels, BatchOperator
prevModel, final int iLabel, final int labelColIdx) {
DataSet<Integer> barrier;
if (prevModel != null) {
barrier = ((DataSet<Row>) prevModel.getDataSet())
.mapPartition(new MapPartitionFunction<Row, Integer>() {
@Override
public void mapPartition(Iterable<Row> values, Collector<Integer> out) throws Exception {
}
});
} else {
barrier = MLEnvironmentFactory.get(data.getMLEnvironmentId()).getExecutionEnvironment().fromElements(0);
}
DataSet<Row> dataSet = data.getDataSet();
dataSet = dataSet
.map(new RichMapFunction<Row, Row>() {
transient Object label;
@Override
public void open(Configuration parameters) throws Exception {
List<Row> bc = getRuntimeContext().getBroadcastVariable("allLabels");
Integer[] order = new Integer[bc.size()];
for (int i = 0; i < order.length; i++) {
order[i] = i;
}
Arrays.sort(order, new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
Comparable v1 = (Comparable) bc.get(o1).getField(0);
Comparable v2 = (Comparable) bc.get(o2).getField(0);
return v1.compareTo(v2);
}
});
if (iLabel >= bc.size()) {
throw new RuntimeException(
"the specified numClasses is larger than the number of distinct labels.: " +
String.format("iLabel = %d, num lables = %d", iLabel, bc.size()));
}
this.label = bc.get(order[iLabel]).getField(0);
}
@Override
public Row map(Row value) throws Exception {
for (int i = 0; i < value.getArity(); i++) {
if (i == labelColIdx) {
if (value.getField(i).equals(label)) {
value.setField(i, 1.0);
} else {
value.setField(i, 0.0);
}
}
}
return value;
}
})
// .withBroadcastSet(barrier, "barrier")
.withBroadcastSet(allLabels.getDataSet(), "allLabels")
.name("CreateTrainData#" + iLabel);
TypeInformation[] colTypes = data.getColTypes().clone();
colTypes[labelColIdx] = Types.DOUBLE;
return new DataSetWrapperBatchOp(dataSet, data.getColNames(), colTypes)
.setMLEnvironmentId(data.getMLEnvironmentId());
}
Example 20
| Source File: GlmTrainBatchOp.java From Alink with Apache License 2.0 | 4 votes |
|
@Override
public GlmTrainBatchOp linkFrom(BatchOperator<?>... inputs) {
BatchOperator<?> in = checkAndGetFirst(inputs);
String[] featureColNames = getFeatureCols();
String labelColName = getLabelCol();
String weightColName = getWeightCol();
String offsetColName = getOffsetCol();
Family familyName = getFamily();
Link linkName = getLink();
double variancePower = getVariancePower();
double linkPower = getLinkPower();
int numIter = getMaxIter();
double epsilon = getEpsilon();
boolean fitIntercept = getFitIntercept();
double regParam = getRegParam();
int numFeature = featureColNames.length;
FamilyLink familyLink = new FamilyLink(familyName, variancePower, linkName, linkPower);
DataSet<Row> data = GlmUtil.preProc(in, featureColNames, offsetColName, weightColName, labelColName);
DataSet<GlmUtil.WeightedLeastSquaresModel> finalModel =
GlmUtil.train(data, numFeature, familyLink, regParam, fitIntercept, numIter, epsilon);
this.setOutput(finalModel.mapPartition(
new BuildModel(featureColNames, offsetColName, weightColName, labelColName,
familyName, variancePower, linkName, linkPower, fitIntercept, numIter, epsilon)).setParallelism(1),
new GlmModelDataConverter().getModelSchema());
//residual
String[] residualColNames = new String[numFeature + 4 + 4];
TypeInformation[] residualColTypes = new TypeInformation[numFeature + 4 + 4];
for (int i = 0; i < numFeature; i++) {
residualColNames[i] = featureColNames[i];
residualColTypes[i] = Types.DOUBLE;
}
residualColNames[numFeature] = "label";
residualColTypes[numFeature] = Types.DOUBLE;
residualColNames[numFeature + 1] = "weight";
residualColTypes[numFeature + 1] = Types.DOUBLE;
residualColNames[numFeature + 2] = "offset";
residualColTypes[numFeature + 2] = Types.DOUBLE;
residualColNames[numFeature + 3] = "pred";
residualColTypes[numFeature + 3] = Types.DOUBLE;
residualColNames[numFeature + 4] = "residualdevianceResiduals";
residualColTypes[numFeature + 4] = Types.DOUBLE;
residualColNames[numFeature + 5] = "pearsonResiduals";
residualColTypes[numFeature + 5] = Types.DOUBLE;
residualColNames[numFeature + 6] = "workingResiduals";
residualColTypes[numFeature + 6] = Types.DOUBLE;
residualColNames[numFeature + 7] = "responseResiduals";
residualColTypes[numFeature + 7] = Types.DOUBLE;
DataSet<Row> residual = GlmUtil.residual(finalModel, data, numFeature, familyLink);
//summary
String[] summaryColNames = new String[1];
TypeInformation[] summaryColTypes = new TypeInformation[1];
summaryColNames[0] = "summary";
summaryColTypes[0] = Types.STRING;
this.setSideOutputTables(new Table[]{
DataSetConversionUtil.toTable(getMLEnvironmentId(),
residual, residualColNames, residualColTypes),
DataSetConversionUtil.toTable(getMLEnvironmentId(), GlmUtil.aggSummary(residual, finalModel,
numFeature, familyLink, regParam, numIter, epsilon, fitIntercept),
summaryColNames, summaryColTypes)
});
return this;
}
