org.apache.flink.api.common.aggregators.DoubleSumAggregator Java Examples

The following examples show how to use org.apache.flink.api.common.aggregators.DoubleSumAggregator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HITS.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void close()
		throws Exception {
	super.close();

	DoubleSumAggregator agg = getIterationRuntimeContext().getIterationAggregator(CHANGE_IN_SCORES);
	agg.aggregate(changeInScores);
}
Example #2
Source File: PageRank.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void close()
		throws Exception {
	super.close();

	DoubleSumAggregator agg = getIterationRuntimeContext().getIterationAggregator(CHANGE_IN_SCORES);
	agg.aggregate(changeInScores);
}
Example #3
Source File: HITS.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void close()
		throws Exception {
	super.close();

	DoubleSumAggregator agg = getIterationRuntimeContext().getIterationAggregator(CHANGE_IN_SCORES);
	agg.aggregate(changeInScores);
}
Example #4
Source File: PageRank.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void close()
		throws Exception {
	super.close();

	DoubleSumAggregator agg = getIterationRuntimeContext().getIterationAggregator(CHANGE_IN_SCORES);
	agg.aggregate(changeInScores);
}
Example #5
Source File: HITS.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void close()
		throws Exception {
	super.close();

	DoubleSumAggregator agg = getIterationRuntimeContext().getIterationAggregator(CHANGE_IN_SCORES);
	agg.aggregate(changeInScores);
}
Example #6
Source File: PageRank.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void close()
		throws Exception {
	super.close();

	DoubleSumAggregator agg = getIterationRuntimeContext().getIterationAggregator(CHANGE_IN_SCORES);
	agg.aggregate(changeInScores);
}
Example #7
Source File: PageRank.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// vertex degree
	DataSet<Vertex<K, Degrees>> vertexDegree = input
		.run(new VertexDegrees<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices)
			.setParallelism(parallelism));

	// vertex count
	DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);

	// s, t, d(s)
	DataSet<Edge<K, LongValue>> edgeSourceDegree = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism))
		.map(new ExtractSourceDegree<>())
			.setParallelism(parallelism)
			.name("Extract source degree");

	// vertices with zero in-edges
	DataSet<Tuple2<K, DoubleValue>> sourceVertices = vertexDegree
		.flatMap(new InitializeSourceVertices<>())
			.setParallelism(parallelism)
			.name("Initialize source vertex scores");

	// s, initial pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> initialScores = vertexDegree
		.map(new InitializeVertexScores<>())
		.withBroadcastSet(vertexCount, VERTEX_COUNT)
			.setParallelism(parallelism)
			.name("Initialize scores");

	IterativeDataSet<Tuple2<K, DoubleValue>> iterative = initialScores
		.iterate(maxIterations)
		.setParallelism(parallelism);

	// s, projected pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> vertexScores = iterative
		.coGroup(edgeSourceDegree)
		.where(0)
		.equalTo(0)
		.with(new SendScore<>())
			.setParallelism(parallelism)
			.name("Send score")
		.groupBy(0)
		.reduce(new SumScore<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum");

	// ignored ID, total pagerank
	DataSet<Tuple2<K, DoubleValue>> sumOfScores = vertexScores
		.reduce(new SumVertexScores<>())
			.setParallelism(parallelism)
			.name("Sum");

	// s, adjusted pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> adjustedScores = vertexScores
		.union(sourceVertices)
			.name("Union with source vertices")
		.map(new AdjustScores<>(dampingFactor))
			.withBroadcastSet(sumOfScores, SUM_OF_SCORES)
			.withBroadcastSet(vertexCount, VERTEX_COUNT)
				.setParallelism(parallelism)
				.name("Adjust scores");

	DataSet<Tuple2<K, DoubleValue>> passThrough;

	if (convergenceThreshold < Double.MAX_VALUE) {
		passThrough = iterative
			.join(adjustedScores)
			.where(0)
			.equalTo(0)
			.with(new ChangeInScores<>())
				.setParallelism(parallelism)
				.name("Change in scores");

		iterative.registerAggregationConvergenceCriterion(CHANGE_IN_SCORES, new DoubleSumAggregator(), new ScoreConvergence(convergenceThreshold));
	} else {
		passThrough = adjustedScores;
	}

	return iterative
		.closeWith(passThrough)
		.map(new TranslateResult<>())
			.setParallelism(parallelism)
			.name("Map result");
}
Example #8
Source File: PageRank.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// vertex degree
	DataSet<Vertex<K, Degrees>> vertexDegree = input
		.run(new VertexDegrees<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices)
			.setParallelism(parallelism));

	// vertex count
	DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);

	// s, t, d(s)
	DataSet<Edge<K, LongValue>> edgeSourceDegree = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism))
		.map(new ExtractSourceDegree<>())
			.setParallelism(parallelism)
			.name("Extract source degree");

	// vertices with zero in-edges
	DataSet<Tuple2<K, DoubleValue>> sourceVertices = vertexDegree
		.flatMap(new InitializeSourceVertices<>())
			.setParallelism(parallelism)
			.name("Initialize source vertex scores");

	// s, initial pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> initialScores = vertexDegree
		.map(new InitializeVertexScores<>())
		.withBroadcastSet(vertexCount, VERTEX_COUNT)
			.setParallelism(parallelism)
			.name("Initialize scores");

	IterativeDataSet<Tuple2<K, DoubleValue>> iterative = initialScores
		.iterate(maxIterations)
		.setParallelism(parallelism);

	// s, projected pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> vertexScores = iterative
		.coGroup(edgeSourceDegree)
		.where(0)
		.equalTo(0)
		.with(new SendScore<>())
			.setParallelism(parallelism)
			.name("Send score")
		.groupBy(0)
		.reduce(new SumScore<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum");

	// ignored ID, total pagerank
	DataSet<Tuple2<K, DoubleValue>> sumOfScores = vertexScores
		.reduce(new SumVertexScores<>())
			.setParallelism(parallelism)
			.name("Sum");

	// s, adjusted pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> adjustedScores = vertexScores
		.union(sourceVertices)
			.name("Union with source vertices")
		.map(new AdjustScores<>(dampingFactor))
			.withBroadcastSet(sumOfScores, SUM_OF_SCORES)
			.withBroadcastSet(vertexCount, VERTEX_COUNT)
				.setParallelism(parallelism)
				.name("Adjust scores");

	DataSet<Tuple2<K, DoubleValue>> passThrough;

	if (convergenceThreshold < Double.MAX_VALUE) {
		passThrough = iterative
			.join(adjustedScores)
			.where(0)
			.equalTo(0)
			.with(new ChangeInScores<>())
				.setParallelism(parallelism)
				.name("Change in scores");

		iterative.registerAggregationConvergenceCriterion(CHANGE_IN_SCORES, new DoubleSumAggregator(), new ScoreConvergence(convergenceThreshold));
	} else {
		passThrough = adjustedScores;
	}

	return iterative
		.closeWith(passThrough)
		.map(new TranslateResult<>())
			.setParallelism(parallelism)
			.name("Map result");
}
Example #9
Source File: PageRank.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// vertex degree
	DataSet<Vertex<K, Degrees>> vertexDegree = input
		.run(new VertexDegrees<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices)
			.setParallelism(parallelism));

	// vertex count
	DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);

	// s, t, d(s)
	DataSet<Edge<K, LongValue>> edgeSourceDegree = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism))
		.map(new ExtractSourceDegree<>())
			.setParallelism(parallelism)
			.name("Extract source degree");

	// vertices with zero in-edges
	DataSet<Tuple2<K, DoubleValue>> sourceVertices = vertexDegree
		.flatMap(new InitializeSourceVertices<>())
			.setParallelism(parallelism)
			.name("Initialize source vertex scores");

	// s, initial pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> initialScores = vertexDegree
		.map(new InitializeVertexScores<>())
		.withBroadcastSet(vertexCount, VERTEX_COUNT)
			.setParallelism(parallelism)
			.name("Initialize scores");

	IterativeDataSet<Tuple2<K, DoubleValue>> iterative = initialScores
		.iterate(maxIterations)
		.setParallelism(parallelism);

	// s, projected pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> vertexScores = iterative
		.coGroup(edgeSourceDegree)
		.where(0)
		.equalTo(0)
		.with(new SendScore<>())
			.setParallelism(parallelism)
			.name("Send score")
		.groupBy(0)
		.reduce(new SumScore<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum");

	// ignored ID, total pagerank
	DataSet<Tuple2<K, DoubleValue>> sumOfScores = vertexScores
		.reduce(new SumVertexScores<>())
			.setParallelism(parallelism)
			.name("Sum");

	// s, adjusted pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> adjustedScores = vertexScores
		.union(sourceVertices)
			.name("Union with source vertices")
		.map(new AdjustScores<>(dampingFactor))
			.withBroadcastSet(sumOfScores, SUM_OF_SCORES)
			.withBroadcastSet(vertexCount, VERTEX_COUNT)
				.setParallelism(parallelism)
				.name("Adjust scores");

	DataSet<Tuple2<K, DoubleValue>> passThrough;

	if (convergenceThreshold < Double.MAX_VALUE) {
		passThrough = iterative
			.join(adjustedScores)
			.where(0)
			.equalTo(0)
			.with(new ChangeInScores<>())
				.setParallelism(parallelism)
				.name("Change in scores");

		iterative.registerAggregationConvergenceCriterion(CHANGE_IN_SCORES, new DoubleSumAggregator(), new ScoreConvergence(convergenceThreshold));
	} else {
		passThrough = adjustedScores;
	}

	return iterative
		.closeWith(passThrough)
		.map(new TranslateResult<>())
			.setParallelism(parallelism)
			.name("Map result");
}
Example #10
Source File: dVMP.java    From toolbox with Apache License 2.0 4 votes vote down vote up 
@Override
public double updateModel(DataFlink<DataInstance> dataUpdate){
    try{
        final ExecutionEnvironment env = dataUpdate.getDataSet().getExecutionEnvironment();

        // get input data
        CompoundVector parameterPrior = this.svb.getNaturalParameterPrior();

        DataSet<CompoundVector> paramSet = env.fromElements(parameterPrior);

        ConvergenceCriterion convergenceELBO;
        if(timeLimit == -1) {
            convergenceELBO = new ConvergenceELBO(this.globalThreshold, System.nanoTime());
        }
        else {
            convergenceELBO = new ConvergenceELBObyTime(this.timeLimit, System.nanoTime());
            this.setMaximumGlobalIterations(5000);
        }
        // set number of bulk iterations for KMeans algorithm
        IterativeDataSet<CompoundVector> loop = paramSet.iterate(maximumGlobalIterations)
                .registerAggregationConvergenceCriterion("ELBO_" + this.getName(), new DoubleSumAggregator(),convergenceELBO);

        Configuration config = new Configuration();
        config.setString(ParameterLearningAlgorithm.BN_NAME, this.getName());
        config.setBytes(SVB, Serialization.serializeObject(svb));

        //We add an empty batched data set to emit the updated prior.
        DataOnMemory<DataInstance> emtpyBatch = new DataOnMemoryListContainer<DataInstance>(dataUpdate.getAttributes());
        DataSet<DataOnMemory<DataInstance>> unionData = null;

        unionData =
                dataUpdate.getBatchedDataSet(this.batchSize, batchConverter)
                        .union(env.fromCollection(Arrays.asList(emtpyBatch),
                                TypeExtractor.getForClass((Class<DataOnMemory<DataInstance>>) Class.forName("eu.amidst.core.datastream.DataOnMemory"))));

        DataSet<CompoundVector> newparamSet =
                unionData
                .map(new ParallelVBMap(randomStart, idenitifableModelling))
                .withParameters(config)
                .withBroadcastSet(loop, "VB_PARAMS_" + this.getName())
                .reduce(new ParallelVBReduce());

        // feed new centroids back into next iteration
        DataSet<CompoundVector> finlparamSet = loop.closeWith(newparamSet);

        parameterPrior = finlparamSet.collect().get(0);

        this.svb.updateNaturalParameterPosteriors(parameterPrior);

        this.svb.updateNaturalParameterPrior(parameterPrior);

        if(timeLimit == -1)
            this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO();
        else
            this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();

        this.svb.applyTransition();

    }catch(Exception ex){
        System.out.println(ex.getMessage().toString());
        ex.printStackTrace();
        throw new RuntimeException(ex.getMessage());
    }

    this.randomStart=false;

    return this.getLogMarginalProbability();
}
Example #11
Source File: ParallelVB.java    From toolbox with Apache License 2.0 4 votes vote down vote up 
public double updateModel(DataFlink<DataInstance> dataUpdate){

        try{
            final ExecutionEnvironment env = dataUpdate.getDataSet().getExecutionEnvironment();

            // get input data
            CompoundVector parameterPrior = this.svb.getNaturalParameterPrior();

            DataSet<CompoundVector> paramSet = env.fromElements(parameterPrior);

            ConvergenceCriterion convergenceELBO;
            if(timeLimit == -1) {
                convergenceELBO = new ConvergenceELBO(this.globalThreshold, System.nanoTime());
            }
            else {
                convergenceELBO = new ConvergenceELBObyTime(this.timeLimit, System.nanoTime());
                this.setMaximumGlobalIterations(5000);
            }
            // set number of bulk iterations for KMeans algorithm
            IterativeDataSet<CompoundVector> loop = paramSet.iterate(maximumGlobalIterations)
                    .registerAggregationConvergenceCriterion("ELBO_" + this.dag.getName(), new DoubleSumAggregator(),convergenceELBO);

            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));

            //We add an empty batched data set to emit the updated prior.
            DataOnMemory<DataInstance> emtpyBatch = new DataOnMemoryListContainer<DataInstance>(dataUpdate.getAttributes());
            DataSet<DataOnMemory<DataInstance>> unionData = null;

            unionData =
                    dataUpdate.getBatchedDataSet(this.batchSize)
                            .union(env.fromCollection(Arrays.asList(emtpyBatch),
                                    TypeExtractor.getForClass((Class<DataOnMemory<DataInstance>>) Class.forName("eu.amidst.core.datastream.DataOnMemory"))));

            DataSet<CompoundVector> newparamSet =
                    unionData
                    .map(new ParallelVBMap(randomStart, idenitifableModelling))
                    .withParameters(config)
                    .withBroadcastSet(loop, "VB_PARAMS_" + this.dag.getName())
                    .reduce(new ParallelVBReduce());

            // feed new centroids back into next iteration
            DataSet<CompoundVector> finlparamSet = loop.closeWith(newparamSet);

            parameterPrior = finlparamSet.collect().get(0);

            this.svb.updateNaturalParameterPosteriors(parameterPrior);

            this.svb.updateNaturalParameterPrior(parameterPrior);

            if(timeLimit == -1)
                this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO();
            else
                this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();

            this.svb.applyTransition();

        }catch(Exception ex){
            throw new RuntimeException(ex.getMessage());
        }

        this.randomStart=false;

        return this.getLogMarginalProbability();
    }
Example #12
Source File: DistributedVI.java    From toolbox with Apache License 2.0 4 votes vote down vote up 
@Override
public double updateModel(DataFlink<DataInstance> dataUpdate){

    try{
        final ExecutionEnvironment env = dataUpdate.getDataSet().getExecutionEnvironment();

        // get input data
        CompoundVector parameterPrior = this.svb.getNaturalParameterPrior();

        DataSet<CompoundVector> paramSet = env.fromElements(parameterPrior);

        ConvergenceCriterion convergenceELBO;
        if(timeLimit == -1) {
            convergenceELBO = new ConvergenceELBO(this.globalThreshold, System.nanoTime());
        }
        else {
            convergenceELBO = new ConvergenceELBObyTime(this.timeLimit, System.nanoTime());
            this.setMaximumGlobalIterations(5000);
        }
        // set number of bulk iterations for KMeans algorithm
        IterativeDataSet<CompoundVector> loop = paramSet.iterate(maximumGlobalIterations)
                .registerAggregationConvergenceCriterion("ELBO_" + this.dag.getName(), new DoubleSumAggregator(),convergenceELBO);

        Configuration config = new Configuration();
        config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
        config.setBytes(SVB, Serialization.serializeObject(svb));

        //We add an empty batched data set to emit the updated prior.
        DataOnMemory<DataInstance> emtpyBatch = new DataOnMemoryListContainer<DataInstance>(dataUpdate.getAttributes());
        DataSet<DataOnMemory<DataInstance>> unionData = null;

        unionData =
                dataUpdate.getBatchedDataSet(this.batchSize)
                        .union(env.fromCollection(Arrays.asList(emtpyBatch),
                                TypeExtractor.getForClass((Class<DataOnMemory<DataInstance>>) Class.forName("eu.amidst.core.datastream.DataOnMemory"))));

        DataSet<CompoundVector> newparamSet =
                unionData
                .map(new ParallelVBMap(randomStart, idenitifableModelling))
                .withParameters(config)
                .withBroadcastSet(loop, "VB_PARAMS_" + this.dag.getName())
                .reduce(new ParallelVBReduce());

        // feed new centroids back into next iteration
        DataSet<CompoundVector> finlparamSet = loop.closeWith(newparamSet);

        parameterPrior = finlparamSet.collect().get(0);

        this.svb.updateNaturalParameterPosteriors(parameterPrior);

        this.svb.updateNaturalParameterPrior(parameterPrior);

        if(timeLimit == -1)
            this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO();
        else
            this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();

        this.svb.applyTransition();

    }catch(Exception ex){
        throw new RuntimeException(ex.getMessage());
    }

    this.randomStart=false;

    return this.getLogMarginalProbability();
}
Example #13
Source File: dVMPv1.java    From toolbox with Apache License 2.0 4 votes vote down vote up 
public double updateModel(DataFlink<DataInstance> dataUpdate){

        try{
            final ExecutionEnvironment env = dataUpdate.getDataSet().getExecutionEnvironment();

            // get input data
            CompoundVector parameterPrior = this.svb.getNaturalParameterPrior();

            DataSet<CompoundVector> paramSet = env.fromElements(parameterPrior);

            ConvergenceCriterion convergenceELBO;
            if(timeLimit == -1) {
                convergenceELBO = new ConvergenceELBO(this.globalThreshold, System.nanoTime());
            }
            else {
                convergenceELBO = new ConvergenceELBObyTime(this.timeLimit, System.nanoTime(), this.idenitifableModelling.getNumberOfEpochs());
                this.setMaximumGlobalIterations(5000);
            }
            // set number of bulk iterations for KMeans algorithm
            IterativeDataSet<CompoundVector> loop = paramSet.iterate(maximumGlobalIterations)
                    .registerAggregationConvergenceCriterion("ELBO_" + this.dag.getName(), new DoubleSumAggregator(),convergenceELBO);

            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));

            //We add an empty batched data set to emit the updated prior.
            DataOnMemory<DataInstance> emtpyBatch = new DataOnMemoryListContainer<DataInstance>(dataUpdate.getAttributes());
            DataSet<DataOnMemory<DataInstance>> unionData = null;

            unionData =
                    dataUpdate.getBatchedDataSet(this.batchSize)
                            .union(env.fromCollection(Arrays.asList(emtpyBatch),
                                    TypeExtractor.getForClass((Class<DataOnMemory<DataInstance>>) Class.forName("eu.amidst.core.datastream.DataOnMemory"))));

            DataSet<CompoundVector> newparamSet =
                    unionData
                    .map(new ParallelVBMap(randomStart, idenitifableModelling))
                    .withParameters(config)
                    .withBroadcastSet(loop, "VB_PARAMS_" + this.dag.getName())
                    .reduce(new ParallelVBReduce());

            // feed new centroids back into next iteration
            DataSet<CompoundVector> finlparamSet = loop.closeWith(newparamSet);

            parameterPrior = finlparamSet.collect().get(0);

            this.svb.updateNaturalParameterPosteriors(parameterPrior);

            this.svb.updateNaturalParameterPrior(parameterPrior);

            if(timeLimit == -1)
                this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO();
            else
                this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();

            this.svb.applyTransition();

        }catch(Exception ex){
            System.out.println(ex.getMessage().toString());
            ex.printStackTrace();
            throw new RuntimeException(ex.getMessage());
        }

        this.randomStart=false;

        return this.getLogMarginalProbability();

    }