Java Code Examples for org.apache.flink.api.java.tuple.Tuple3#of()

The following examples show how to use org.apache.flink.api.java.tuple.Tuple3#of() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DocCountVectorizerTrainBatchOp.java    From Alink with Apache License 2.0 6 votes vote down vote up
@Override
        public void mapPartition(Iterable<Tuple2<Long, Row>> iterable, Collector<DocCountVectorizerModelData> collector) throws Exception {
            List<String> data = new ArrayList<>();
            Tuple3<String, Double, Integer> feature = Tuple3.of(null, null, null);
            for (Tuple2<Long, Row> tuple : iterable) {
                Row row = tuple.f1;

                feature.f0 = row.getField(0).toString();
                feature.f1 = ((Number)row.getField(2)).doubleValue();
                feature.f2 = tuple.f0.intValue();

                data.add(JsonConverter.toJson(feature));
            }

            DocCountVectorizerModelData modelData = new DocCountVectorizerModelData();
            modelData.featureType = featureType;
            modelData.minTF = minTF;
            modelData.list = data;
            collector.collect(modelData);

//            new DocCountVectorizerModelDataConverter().save(modelData, collector);
        }
 
Example 2
Source File: PrefixSpanBatchOp.java    From Alink with Apache License 2.0 6 votes vote down vote up
/**
 * Encode the sequence patterns.
 */
private static Tuple3<String, Long, Long> encodeSequence(int[] sequence, String[] indexToString) {
    StringBuilder sbd = new StringBuilder();
    int itemSetSize = 0;
    long chainLength = 1L;
    long itemCount = 0L;
    for (int i = 1; i < sequence.length - 1; i++) {
        if (sequence[i] == 0) {
            sbd.append(ELEMENT_SEPARATOR);
            chainLength++;
            itemSetSize = 0;
        } else {
            if (itemSetSize > 0) {
                sbd.append(ITEM_SEPARATOR);
            }
            sbd.append(indexToString[sequence[i]]);
            itemSetSize++;
            itemCount++;
        }
    }
    return Tuple3.of(sbd.toString(), itemCount, chainLength);
}
 
Example 3
Source File: VectorMaxAbsScalerModelDataConverter.java    From Alink with Apache License 2.0 6 votes vote down vote up
/**
 * Serialize the model data to "Tuple3<Params, List<String>, List<Row>>".
 *
 * @param modelData The model data to serialize.
 * @return The serialization result.
 */
public Tuple3<Params, Iterable<String>, Iterable<Row>> serializeModel(BaseVectorSummary modelData) {

    double[] maxData;
    double[] minData;
    if (modelData.max() instanceof DenseVector) {
        maxData = ((DenseVector) modelData.max()).getData();
    } else {
        maxData = ((SparseVector) modelData.max()).toDenseVector().getData();
    }
    if (modelData.min() instanceof DenseVector) {
        minData = ((DenseVector) modelData.min()).getData();
    } else {
        minData = ((SparseVector) modelData.min()).toDenseVector().getData();
    }

    double[] maxAbs = new double[maxData.length];
    for (int i = 0; i < maxAbs.length; i++) {
        maxAbs[i] = Math.max(Math.abs(minData[i]), Math.abs(maxData[i]));
    }

    List<String> data = new ArrayList<>();
    data.add(JsonConverter.toJson(maxAbs));

    return Tuple3.of(new Params(), data, new ArrayList<>());
}
 
Example 4
Source File: QueryableWindowOperatorEvicting.java    From yahoo-streaming-benchmark with Apache License 2.0 6 votes vote down vote up
@Override
public void processWatermark(Watermark watermark) throws Exception {

	StreamRecord<Tuple3<String, Long, Long>> result = new StreamRecord<>(null, -1);

	Iterator<Map.Entry<Long, Map<UUID, CountAndAccessTime>>> iterator = windows.entrySet().iterator();
	
	while (iterator.hasNext()) {
		Map.Entry<Long, Map<UUID, CountAndAccessTime>> entry = iterator.next();
		
		if (entry.getKey() < watermark.getTimestamp()) {
			iterator.remove();
			Long windowTimestamp = entry.getKey();
			
			// emit window
			for (Map.Entry<UUID, CountAndAccessTime> window : entry.getValue().entrySet()) {
				Tuple3<String, Long, Long> resultTuple = Tuple3.of(window.getKey().toString(), windowTimestamp, window.getValue().count);
				output.collect(result.replace(resultTuple));
			}
		}
	}
	
	lastWatermark = watermark.getTimestamp();
}
 
Example 5
Source File: CloudHBaseSinkFunctionExample.java    From alibaba-flink-connectors with Apache License 2.0 5 votes vote down vote up
private Tuple3<Boolean, Long, Row> getDeleteTuple() {
	ThreadLocalRandom random = ThreadLocalRandom.current();
	long rowKey = random.nextLong();
	Row row = new Row(columnNames.getArity());
	for (int i = 0; i < columnNames.getArity(); i++) {
		if (random.nextBoolean()) {
			row.setField(i, true);
		}
	}
	return Tuple3.of(false, rowKey, row);
}
 
Example 6
Source File: Stacker.java    From Alink with Apache License 2.0 5 votes vote down vote up
public Tuple3<Double, Double, Vector> stack(List<Tuple2<Double, DenseVector>> batchData, int count) {
    int l = inputSize * count + count;
    DenseVector stacked = new DenseVector(l);
    int offset = 0;
    for (int i = 0; i < count; i++) {
        System.arraycopy(batchData.get(i).f1.getData(), 0, stacked.getData(), offset, inputSize);
        offset += inputSize;
    }
    for (int i = 0; i < count; i++) {
        stacked.set(offset, batchData.get(i).f0);
        offset++;
    }
    return Tuple3.of((double) count, 0., stacked);
}
 
Example 7
Source File: NaiveBayesTextModelDataConverter.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
 * Serialize the model data to "Tuple3<Params, List<String>, List<Object>>".
 *
 * @param modelData The model data to serialize.
 * @return The serialization result.
 */
@Override
public Tuple3<Params, Iterable<String>, Iterable<Object>> serializeModel(NaiveBayesTextTrainModelData modelData) {
    Params meta = new Params()
            .set(NaiveBayesTextTrainParams.MODEL_TYPE, ParamUtil.searchEnum(NaiveBayesTextTrainParams.MODEL_TYPE, modelData.modelType.name()))
            .set(HasVectorCol.VECTOR_COL, modelData.vectorColName);
    NaiveBayesTextProbInfo data = new NaiveBayesTextProbInfo();
    data.piArray = modelData.pi;
    data.theta = modelData.theta;
    return Tuple3.of(meta, Collections.singletonList(JsonConverter.toJson(data)), Arrays.asList(modelData.label));
}
 
Example 8
Source File: LinearModelDataConverter.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
 * Serialize the model data to "Tuple3<Params, List<String>, Iterable<Object>>".
 *
 * @param modelData The model data to serialize.
 * @return The serialization result.
 */
@Override
public Tuple3<Params, Iterable<String>, Iterable<Object>> serializeModel(LinearModelData modelData) {
    List<String> data = new ArrayList<>();
    data.add(JsonConverter.toJson(getModelData(modelData)));
    return Tuple3.of(getMetaInfo(modelData), data,
        modelData.labelValues == null? null: Arrays.asList(modelData.labelValues));
}
 
Example 9
Source File: ImputerModelDataConverter.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
 * Serialize the model to "Tuple3<Params, List<String>, List<Row>>"
 *
 * @param modelData The model data to serialize.
 * @return The serialization result.
 */
@Override
public Tuple3<Params, Iterable<String>, Iterable<Row>> serializeModel(Tuple3<Strategy, TableSummary, String> modelData) {
    Strategy strategy = modelData.f0;
    TableSummary summary = modelData.f1;
    String fillValue = modelData.f2;

    double[] values = null;
    Params meta = new Params()
            .set(STRATEGY, strategy)
            .set(SELECTED_COLS, selectedColNames);
    switch (strategy) {
        case MIN:
            values = new double[selectedColNames.length];
            for (int i = 0; i < selectedColNames.length; i++) {
                values[i] = summary.min(selectedColNames[i]);
            }
            break;
        case MAX:
            values = new double[selectedColNames.length];
            for (int i = 0; i < selectedColNames.length; i++) {
                values[i] = summary.max(selectedColNames[i]);
            }
            break;
        case MEAN:
            values = new double[selectedColNames.length];
            for (int i = 0; i < selectedColNames.length; i++) {
                values[i] = summary.mean(selectedColNames[i]);
            }
            break;
        default:
            meta.set(FILL_VALUE, fillValue);
    }

    List<String> data = new ArrayList<>();
    data.add(JsonConverter.toJson(values));

    return Tuple3.of(meta, data, new ArrayList<>());
}
 
Example 10
Source File: VectorImputerModelDataConverter.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
 * Deserialize the model data.
 *
 * @param meta         The model meta data.
 * @param data         The model concrete data.
 * @param additionData The additional data.
 * @return The model data used by mapper.
 */
@Override
public Tuple3<Strategy, double[], Double> deserializeModel(Params meta, Iterable<String> data, Iterable<Row> additionData) {
    Strategy strategy = meta.get(STRATEGY);
    double[] values = null;
    if(data.iterator().hasNext()) {
        values = JsonConverter.fromJson(data.iterator().next(), double[].class);
    }
    Double fillValue = meta.get(FILL_VALUE);
    return Tuple3.of(strategy, values, fillValue);
}
 
Example 11
Source File: VectorImputerModelDataConverter.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
 * Serialize the model data to "Tuple3<Params, List<String>, List<Row>>".
 *
 * @param modelData The model data to serialize.
 * @return The serialization result.
 */
public Tuple3<Params, Iterable<String>, Iterable<Row>> serializeModel(Tuple3<Strategy, BaseVectorSummary, Double> modelData) {
    Strategy strategy = modelData.f0;
    BaseVectorSummary summary = modelData.f1;
    double fillValue = modelData.f2;
    double[] values = null;
    Params meta = new Params()
            .set(SELECTED_COL, vectorColName)
            .set(STRATEGY, strategy);
    switch (strategy) {
        case MIN:
            if (summary.min() instanceof DenseVector) {
                values = ((DenseVector) summary.min()).getData();
            } else {
                values = ((SparseVector) summary.min()).toDenseVector().getData();
            }
            break;
        case MAX:
            if (summary.max() instanceof DenseVector) {
                values = ((DenseVector) summary.max()).getData();
            } else {
                values = ((SparseVector) summary.max()).toDenseVector().getData();
            }
            break;
        case MEAN:
            if (summary.mean() instanceof DenseVector) {
                values = ((DenseVector) summary.mean()).getData();
            } else {
                values = ((SparseVector) summary.mean()).getValues();
            }
            break;
        default:
            meta.set(FILL_VALUE, fillValue);
    }
    List<String> data = new ArrayList<>();
    data.add(JsonConverter.toJson(values));

    return Tuple3.of(meta, data, new ArrayList<>());
}
 
Example 12
Source File: CloudHBaseSinkFunctionExample.java    From alibaba-flink-connectors with Apache License 2.0 5 votes vote down vote up
private Tuple3<Boolean, Long, Row> getPutTuple() {
	ThreadLocalRandom random = ThreadLocalRandom.current();
	long rowKey = random.nextLong();
	Row row = new Row(columnNames.getArity());
	for (int i = 0; i < columnNames.getArity(); i++) {
		row.setField(i, random.nextDouble());
	}
	return Tuple3.of(true, rowKey, row);
}
 
Example 13
Source File: VectorStandardScalerModelDataConverter.java    From Alink with Apache License 2.0 4 votes vote down vote up
/**
 * Serialize the model data to "Tuple3<Params, List<String>, List<Row>>".
 *
 * @param modelData The model data to serialize.
 * @return The serialization result.
 */
public Tuple3<Params, Iterable<String>, Iterable<Row>> serializeModel(Tuple3<Boolean, Boolean, BaseVectorSummary> modelData) {
    Boolean withMean = modelData.f0;
    Boolean withStd = modelData.f1;
    BaseVectorSummary summary = modelData.f2;

    double[] means;
    double[] stdDeviations;

    int n = summary.vectorSize();
    if (withMean) {
        if (summary.mean() instanceof DenseVector) {
            means = ((DenseVector) summary.mean()).getData();
        } else {
            means = ((SparseVector) summary.mean()).toDenseVector().getData();
        }
    } else {
        means = new double[n];
    }
    if (withStd) {
        if (summary.standardDeviation() instanceof DenseVector) {
            stdDeviations = ((DenseVector) summary.standardDeviation()).getData();
        } else {
            stdDeviations = ((SparseVector) summary.standardDeviation()).toDenseVector().getData();
        }
    } else {
        stdDeviations = new double[n];
        Arrays.fill(stdDeviations, 1);
    }

    List<String> data = new ArrayList<>();
    data.add(JsonConverter.toJson(means));
    data.add(JsonConverter.toJson(stdDeviations));

    Params meta = new Params()
        .set(VectorStandardTrainParams.WITH_MEAN, withMean)
        .set(VectorStandardTrainParams.WITH_STD, withStd)
        .set(VectorMinMaxScalerTrainParams.SELECTED_COL, vectorColName);

    return Tuple3.of(meta, data, new ArrayList<>());
}
 
Example 14
Source File: BinaryClassMetricsTest.java    From Alink with Apache License 2.0 4 votes vote down vote up
@Test
public void sampleTest(){
    EvaluationCurve[] curves = new EvaluationCurve[3];
    ConfusionMatrix[] data = new ConfusionMatrix[7];

    double[] threshold = new double[] {1.0, 0.9, 0.8, 0.75, 0.7, 0.6, 0.5};

    long[][] matrix = new long[][] {{0L, 0L}, {3L, 2L}, {1L, 0L}, {2L, 2L}, {2L, 0L}, {1L, 2L}, {2L, 1L}, {1L, 1L},
        {3L, 1L},
        {0L, 1L}, {3L, 2L}, {0L, 0L}, {3L, 2L}, {0L, 0L}};

    int cnt = 0;
    for (int i = 0; i < data.length; i++) {
        data[i] = new ConfusionMatrix(new long[][] {matrix[cnt], matrix[cnt + 1]});
        cnt += 2;
    }

    double[][] curve = new double[][] {
        {0.0, 0.0, 0.0, 0.5, 0.5, 1.0, 1.0},
        {0.0, 0.333, 0.666, 0.666, 1.0, 1.0, 1.0},
        {0.0, 0.333, 0.666, 0.666, 1.0, 1.0, 1.0},
        {1.0, 1.0, 1.0, 0.666, 0.75, 0.6, 0.6},
        {0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0},
        {0.0, 1.0, 2.0, 2.0, 3.0, 3.0, 3.0}};

    cnt = 0;
    for (int i = 0; i < curves.length; i++) {
        EvaluationCurvePoint[] curvePoints = new EvaluationCurvePoint[7];
        for (int j = 0; j < threshold.length; j++) {
            curvePoints[j] = new EvaluationCurvePoint(curve[cnt][j], curve[cnt + 1][j], threshold[j]);
        }
        curves[i] = new EvaluationCurve(curvePoints);

        cnt += 2;
    }

    Tuple3<ConfusionMatrix[], double[], EvaluationCurve[]> res = Tuple3.of(data, threshold, curves);
    Tuple3<ConfusionMatrix[], double[], EvaluationCurve[]> sampleRes = sample(0.1, res);
    double[] expect = new double[] {0.9, 0.8, 0.7, 0.6, 0.5};
    Assert.assertEquals(sampleRes.f0.length, expect.length);
    for (int i = 0; i < expect.length; i++) {
        Assert.assertEquals(expect[i], sampleRes.f1[i], 0.001);
    }
}
 
Example 15
Source File: LocalitySensitiveHashApproxFunctions.java    From Alink with Apache License 2.0 4 votes vote down vote up
@Override
public Tuple3 <Vector, Long, Row> map(Tuple2 <Long, Row> tuple) {
	Row row = tuple.f1;
	Vector vec = VectorUtil.getVector(row.getField(1));
	return Tuple3.of(vec, tuple.f0, Row.of(row.getField(0)));
}
 
Example 16
Source File: TreeModelDataConverter.java    From Alink with Apache License 2.0 4 votes vote down vote up
@Override
protected Tuple3<Params, Iterable<String>, Iterable<Object>> serializeModel(TreeModelDataConverter modelData) {
	List <String> serialized = new ArrayList <>();

	int pStart = 0, pEnd = 0;

	// toString partition of stringIndexerModel
	if (modelData.stringIndexerModelSerialized != null) {
		for (Row row : stringIndexerModelSerialized) {
			Object[] objs = new Object[row.getArity()];

			for (int i = 0; i < row.getArity(); ++i) {
				objs[i] = row.getField(i);
			}

			serialized.add(JsonConverter.toJson(objs));
		}

		pEnd = serialized.size();
	}

	modelData.meta.set(STRING_INDEXER_MODEL_PARTITION, Partition.of(pStart, pEnd));

	// toString partition of trees
	Partitions treesPartition = new Partitions();

	for (Node root : modelData.roots) {
		List <String> localSerialize = serializeTree(root);

		pStart = pEnd;
		pEnd = pStart + localSerialize.size();

		treesPartition.add(Partition.of(pStart, pEnd));

		serialized.addAll(localSerialize);
	}

	modelData.meta.set(TREE_PARTITIONS, treesPartition);

	return Tuple3.of(
		modelData.meta,
		serialized,
		modelData.labels == null ? null : Arrays.asList(modelData.labels)
	);
}
 
Example 17
Source File: BinaryMetricsSummary.java    From Alink with Apache License 2.0 4 votes vote down vote up
/**
 * Extract the bins who are not empty, keep the middle threshold 0.5.
 * <p>
 * Initialize the RocCurve, Recall-Precision Curve and Lift Curve.
 * <p>
 * RocCurve: (FPR, TPR), starts with (0,0). Recall-Precision Curve: (recall, precision), starts with (0, p), p is
 * the precision with the lowest. LiftChart: (TP+FP/total, TP), starts with (0,0). confusion matrix = [TP FP][FN
 * TN].
 *
 * @param positiveBin positiveBins.
 * @param negativeBin negativeBins.
 * @param total       sample number
 * @return ConfusionMatrix array, threshold array, rocCurve/recallPrecisionCurve/LiftChart.
 */
static Tuple3<ConfusionMatrix[], double[], EvaluationCurve[]> extractMatrixThreCurve(long[] positiveBin,
                                                                                     long[] negativeBin,
                                                                                     long total) {
    ArrayList<Integer> effectiveIndices = new ArrayList<>();
    long totalTrue = 0, totalFalse = 0;
    for (int i = 0; i < ClassificationEvaluationUtil.DETAIL_BIN_NUMBER; i++) {
        if (0L != positiveBin[i] || 0L != negativeBin[i]
            || i == ClassificationEvaluationUtil.DETAIL_BIN_NUMBER / 2) {
            effectiveIndices.add(i);
            totalTrue += positiveBin[i];
            totalFalse += negativeBin[i];
        }
    }
    Preconditions.checkState(totalFalse + totalTrue == total,
        "The effective number in bins must be equal to total!");

    final int length = effectiveIndices.size();
    final int newLen = length + 1;
    final double m = 1.0 / ClassificationEvaluationUtil.DETAIL_BIN_NUMBER;
    EvaluationCurvePoint[] rocCurve = new EvaluationCurvePoint[newLen];
    EvaluationCurvePoint[] recallPrecisionCurve = new EvaluationCurvePoint[newLen];
    EvaluationCurvePoint[] liftChart = new EvaluationCurvePoint[newLen];
    ConfusionMatrix[] data = new ConfusionMatrix[newLen];
    double[] threshold = new double[newLen];
    long curTrue = 0;
    long curFalse = 0;
    for (int i = 1; i < newLen; i++) {
        int index = effectiveIndices.get(length - i);
        curTrue += positiveBin[index];
        curFalse += negativeBin[index];
        threshold[i] = index * m;
        data[i] = new ConfusionMatrix(
            new long[][] {{curTrue, curFalse}, {totalTrue - curTrue, totalFalse - curFalse}});
        double tpr = (totalTrue == 0 ? 1.0 : 1.0 * curTrue / totalTrue);
        rocCurve[i] = new EvaluationCurvePoint(totalFalse == 0 ? 1.0 : 1.0 * curFalse / totalFalse, tpr,
            threshold[i]);
        recallPrecisionCurve[i] = new EvaluationCurvePoint(tpr, curTrue + curTrue == 0 ? 1.0 : 1.0 * curTrue / (curTrue + curFalse), threshold[i]);
        liftChart[i] = new EvaluationCurvePoint(1.0 * (curTrue + curFalse) / total, curTrue, threshold[i]);
    }

    threshold[0] = 1.0;
    data[0] = new ConfusionMatrix(new long[][] {{0, 0}, {totalTrue, totalFalse}});
    rocCurve[0] = new EvaluationCurvePoint(0, 0, threshold[0]);
    recallPrecisionCurve[0] = new EvaluationCurvePoint(0, recallPrecisionCurve[1].getY(), threshold[0]);
    liftChart[0] = new EvaluationCurvePoint(0, 0, threshold[0]);

    return Tuple3.of(data, threshold, new EvaluationCurve[] {new EvaluationCurve(rocCurve),
        new EvaluationCurve(recallPrecisionCurve), new EvaluationCurve(liftChart)});
}
 
Example 18
Source File: ValueStateTransformationTest.java    From bravo with Apache License 2.0 4 votes vote down vote up
@Override
public Tuple3<Integer, Integer, Integer> map(Integer value) throws Exception {
	count.update(Optional.ofNullable(count.value()).orElse(0) + 1);
	count2.update(Optional.ofNullable(count2.value()).orElse(0) + 1);
	return Tuple3.of(value, count.value(), count2.value());
}
 
Example 19
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that accepted slots go into state assigned and the others are returned to the resource
 * manager.
 */
@Test
public void testSlotAcceptance() throws Exception {
	final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(mock(ResourceProfile.class), mock(ResourceProfile.class)), timerService);
	final JobManagerTable jobManagerTable = new JobManagerTable();
	final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());

	final String resourceManagerAddress = "rm";
	final UUID resourceManagerLeaderId = UUID.randomUUID();

	final String jobManagerAddress = "jm";
	final UUID jobManagerLeaderId = UUID.randomUUID();

	resourceManagerLeaderRetriever.notifyListener(resourceManagerAddress, resourceManagerLeaderId);
	jobManagerLeaderRetriever.notifyListener(jobManagerAddress, jobManagerLeaderId);

	final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	final ResourceID resourceManagerResourceId = resourceManagerGateway.getOwnResourceId();
	final InstanceID registrationId = new InstanceID();

	final CompletableFuture<ResourceID> registrationFuture = new CompletableFuture<>();
	resourceManagerGateway.setRegisterTaskExecutorFunction(
		stringResourceIDIntegerHardwareDescriptionTuple4 -> {
			registrationFuture.complete(stringResourceIDIntegerHardwareDescriptionTuple4.f1);
			return CompletableFuture.completedFuture(new TaskExecutorRegistrationSuccess(registrationId, resourceManagerResourceId, new ClusterInformation("localhost", 1234)));
		});

	final CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture = new CompletableFuture<>();
	resourceManagerGateway.setNotifySlotAvailableConsumer(availableSlotFuture::complete);

	final ResourceID jmResourceId = new ResourceID(jobManagerAddress);

	final AllocationID allocationId1 = new AllocationID();
	final AllocationID allocationId2 = new AllocationID();

	final SlotOffer offer1 = new SlotOffer(allocationId1, 0, ResourceProfile.UNKNOWN);

	final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);

	when(jobMasterGateway.registerTaskManager(
			any(String.class),
			eq(taskManagerLocation),
			any(Time.class)
	)).thenReturn(CompletableFuture.completedFuture(new JMTMRegistrationSuccess(jmResourceId)));
	when(jobMasterGateway.getHostname()).thenReturn(jobManagerAddress);

	when(jobMasterGateway.offerSlots(
			any(ResourceID.class), any(Collection.class), any(Time.class)))
		.thenReturn(CompletableFuture.completedFuture((Collection<SlotOffer>) Collections.singleton(offer1)));

	rpc.registerGateway(resourceManagerAddress, resourceManagerGateway);
	rpc.registerGateway(jobManagerAddress, jobMasterGateway);

	final TaskExecutorLocalStateStoresManager localStateStoresManager = createTaskExecutorLocalStateStoresManager();

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskManagerLocation(taskManagerLocation)
		.setTaskSlotTable(taskSlotTable)
		.setJobManagerTable(jobManagerTable)
		.setJobLeaderService(jobLeaderService)
		.setTaskStateManager(localStateStoresManager)
		.build();

	TaskExecutor taskManager = createTaskExecutor(taskManagerServices);

	try {
		taskManager.start();

		assertThat(registrationFuture.get(), equalTo(taskManagerLocation.getResourceID()));

		taskSlotTable.allocateSlot(0, jobId, allocationId1, Time.milliseconds(10000L));
		taskSlotTable.allocateSlot(1, jobId, allocationId2, Time.milliseconds(10000L));

		// we have to add the job after the TaskExecutor, because otherwise the service has not
		// been properly started.
		jobLeaderService.addJob(jobId, jobManagerAddress);

		final Tuple3<InstanceID, SlotID, AllocationID> instanceIDSlotIDAllocationIDTuple3 = availableSlotFuture.get();

		final Tuple3<InstanceID, SlotID, AllocationID> expectedResult = Tuple3.of(registrationId, new SlotID(taskManagerLocation.getResourceID(), 1), allocationId2);

		assertThat(instanceIDSlotIDAllocationIDTuple3, equalTo(expectedResult));

		assertTrue(taskSlotTable.tryMarkSlotActive(jobId, allocationId1));
		assertFalse(taskSlotTable.tryMarkSlotActive(jobId, allocationId2));
		assertTrue(taskSlotTable.isSlotFree(1));
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}
}
 
Example 20
Source File: MlpcModelDataConverter.java    From Alink with Apache License 2.0 4 votes vote down vote up
@Override
protected Tuple3<Params, Iterable<String>, Iterable<Object>> serializeModel(MlpcModelData modelData) {
    List<String> data = new ArrayList<>();
    data.add(JsonConverter.toJson(modelData.weights));
    return Tuple3.of(modelData.meta, data, modelData.labels);
}