Java Code Examples for org.apache.flink.configuration.Configuration#setBytes()

The following examples show how to use org.apache.flink.configuration.Configuration#setBytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: dVMPv1.java    From toolbox with Apache License 2.0 6 votes vote down vote up 
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
Example 2
Source File: ParallelVB.java    From toolbox with Apache License 2.0 6 votes vote down vote up 
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
Example 3
Source File: DistributedVI.java    From toolbox with Apache License 2.0 6 votes vote down vote up 
public DataSet<DataPosterior> computePosterior(){

        Attribute seq_id = this.dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));

            return this.dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
Example 4
Source File: ParallelVB.java    From toolbox with Apache License 2.0 6 votes vote down vote up 
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink, List<Variable> latentVariables){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
Example 5
Source File: ParallelVB.java    From toolbox with Apache License 2.0 6 votes vote down vote up 
public DataSet<DataPosteriorAssignment> computePosteriorAssignment(DataFlink<DataInstance> dataFlink, List<Variable> latentVariables){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInferenceAssignment())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
Example 6
Source File: dVMPv1.java    From toolbox with Apache License 2.0 6 votes vote down vote up 
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink, List<Variable> latentVariables){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
Example 7
Source File: DistributedVI.java    From toolbox with Apache License 2.0 6 votes vote down vote up 
public DataSet<DataPosterior> computePosterior(List<Variable> latentVariables){

        Attribute seq_id = this.dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return this.dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
Example 8
Source File: dVMP.java    From toolbox with Apache License 2.0 6 votes vote down vote up 
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink, List<Variable> latentVariables){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return dataFlink
                    .getBatchedDataSet(this.batchSize,batchConverter)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
Example 9
Source File: ParallelMaximumLikelihood2.java    From toolbox with Apache License 2.0 5 votes vote down vote up 
/**
 * {@inheritDoc}
 */
@Override
public double updateModel(DataFlink<DataInstance> dataUpdate) {

    try {

        this.initLearning();

        Configuration config = new Configuration();
        config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
        config.setBytes(EFBN_NAME, Serialization.serializeObject(efBayesianNetwork));

        DataSet<DataInstance> dataset = dataUpdate.getDataSet();

        this.sumSS = dataset.mapPartition(new SufficientSatisticsMAP())
                .withParameters(config)
                .reduce(new SufficientSatisticsReduce())
                .collect().get(0);

        //Add the prior
        sumSS.sum(efBayesianNetwork.createInitSufficientStatistics());

        JobExecutionResult result = dataset.getExecutionEnvironment().getLastJobExecutionResult();

        numInstances = result.getAccumulatorResult(ParallelMaximumLikelihood2.COUNTER_NAME+"_"+this.dag.getName());
        numInstances++;//Initial counts

    }catch(Exception ex){
        throw new UndeclaredThrowableException(ex);
    }

    return this.getLogMarginalProbability();
}
Example 10
Source File: ParallelMaximumLikelihood.java    From toolbox with Apache License 2.0 5 votes vote down vote up 
/**
 * {@inheritDoc}
 */
@Override
public double updateModel(DataFlink<DataInstance> dataUpdate) {
    try {
        Configuration config = new Configuration();
        config.setString(BN_NAME, this.dag.getName());
        config.setBytes(EFBN_NAME, Serialization.serializeObject(efBayesianNetwork));

        DataSet<DataInstance> dataset = dataUpdate.getDataSet();
        this.sumSS = dataset.map(new SufficientSatisticsMAP())
                .withParameters(config)
                .reduce(new SufficientSatisticsReduce())
                .collect().get(0);

        //Add the prior
        sumSS.sum(efBayesianNetwork.createInitSufficientStatistics());

        JobExecutionResult result = dataset.getExecutionEnvironment().getLastJobExecutionResult();

        numInstances = result.getAccumulatorResult(ParallelMaximumLikelihood.COUNTER_NAME+"_"+this.dag.getName());
        numInstances++;//Initial counts

    }catch(Exception ex){
        throw new UndeclaredThrowableException(ex);
    }

    return this.getLogMarginalProbability();
}
Example 11
Source File: StochasticVI.java    From toolbox with Apache License 2.0 5 votes vote down vote up 
public static double computeELBO(DataFlink<DataInstance> dataFlink, SVB svb, Function2<DataFlink<DataInstance>,Integer,DataSet<DataOnMemory<DataInstance>>> batchConverter){

        svb.setOutput(false);
        double elbo =  svb.getPlateuStructure().getNonReplictedNodes().mapToDouble(node -> svb.getPlateuStructure().getVMP().computeELBO(node)).sum();

        try {

            Configuration config = new Configuration();
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(PRIOR, Serialization.serializeObject(svb.getPlateuStructure().getPlateauNaturalParameterPosterior()));

            DataSet<DataOnMemory<DataInstance>> batches;
            if (batchConverter!=null)
                batches= dataFlink.getBatchedDataSet(svb.getWindowsSize(),batchConverter);
            else
                batches= dataFlink.getBatchedDataSet(svb.getWindowsSize());

            elbo += batches.map(new ParallelVBMapELBO())
                    .withParameters(config)
                    .reduce(new ReduceFunction<Double>() {
                        @Override
                        public Double reduce(Double aDouble, Double t1) throws Exception {
                            return aDouble + t1;
                        }
                    }).collect().get(0);
        } catch (Exception e) {
            e.printStackTrace();
        }

        svb.setOutput(true);

        return elbo;
    }
Example 12
Source File: DistributedCache.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
	int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
	conf.setInteger(CACHE_FILE_NUM, num);
	conf.setString(CACHE_FILE_NAME + num, name);
	conf.setString(CACHE_FILE_PATH + num, e.filePath);
	conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
	conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
	if (e.blobKey != null) {
		conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
	}
}
Example 13
Source File: ConversionToBatches.java    From toolbox with Apache License 2.0 5 votes vote down vote up 
public static <T extends DataInstance> DataSet<DataOnMemory<T>> toBatchesBySeqID(DataFlink<T> data, int batchSize){

        try{
            Configuration config = new Configuration();
            config.setInteger(BATCH_SIZE, batchSize);
            config.setBytes(ATTRIBUTES, Serialization.serializeObject(data.getAttributes()));

            return data.getDataSet().mapPartition(new DataBatchBySeqID<T>()).withParameters(config);
        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }
    }
Example 14
Source File: ConversionToBatches.java    From toolbox with Apache License 2.0 5 votes vote down vote up 
public static <T extends DataInstance> DataSet<DataOnMemory<T>> toBatches(DataFlink<T> data, int batchSize){

        try{
            Configuration config = new Configuration();
            config.setInteger(BATCH_SIZE, batchSize);
            config.setBytes(ATTRIBUTES, Serialization.serializeObject(data.getAttributes()));

            return data.getDataSet().mapPartition(new DataBatch<T>()).withParameters(config);
        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }
    }
Example 15
Source File: DistributedCache.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
	int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
	conf.setInteger(CACHE_FILE_NUM, num);
	conf.setString(CACHE_FILE_NAME + num, name);
	conf.setString(CACHE_FILE_PATH + num, e.filePath);
	conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
	conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
	if (e.blobKey != null) {
		conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
	}
}
Example 16
Source File: ConfigurationUtil.java    From stateful-functions with Apache License 2.0 5 votes vote down vote up 
public static void storeSerializedInstance(
    Configuration configuration, ConfigOption<byte[]> option, Object instance) {
  try {
    byte[] bytes = InstantiationUtil.serializeObject(instance);
    configuration.setBytes(option.key(), bytes);
  } catch (IOException e) {
    throw new IllegalStateException(e);
  }
}
Example 17
Source File: DistributedCache.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
	int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
	conf.setInteger(CACHE_FILE_NUM, num);
	conf.setString(CACHE_FILE_NAME + num, name);
	conf.setString(CACHE_FILE_PATH + num, e.filePath);
	conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
	conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
	if (e.blobKey != null) {
		conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
	}
}
Example 18
Source File: InstantiationUtil.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static void writeObjectToConfig(Object o, Configuration config, String key) throws IOException {
	byte[] bytes = serializeObject(o);
	config.setBytes(key, bytes);
}
Example 19
Source File: DistributedVI.java    From toolbox with Apache License 2.0 4 votes vote down vote up 
@Override
public double updateModel(DataFlink<DataInstance> dataUpdate){

    try{
        final ExecutionEnvironment env = dataUpdate.getDataSet().getExecutionEnvironment();

        // get input data
        CompoundVector parameterPrior = this.svb.getNaturalParameterPrior();

        DataSet<CompoundVector> paramSet = env.fromElements(parameterPrior);

        ConvergenceCriterion convergenceELBO;
        if(timeLimit == -1) {
            convergenceELBO = new ConvergenceELBO(this.globalThreshold, System.nanoTime());
        }
        else {
            convergenceELBO = new ConvergenceELBObyTime(this.timeLimit, System.nanoTime());
            this.setMaximumGlobalIterations(5000);
        }
        // set number of bulk iterations for KMeans algorithm
        IterativeDataSet<CompoundVector> loop = paramSet.iterate(maximumGlobalIterations)
                .registerAggregationConvergenceCriterion("ELBO_" + this.dag.getName(), new DoubleSumAggregator(),convergenceELBO);

        Configuration config = new Configuration();
        config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
        config.setBytes(SVB, Serialization.serializeObject(svb));

        //We add an empty batched data set to emit the updated prior.
        DataOnMemory<DataInstance> emtpyBatch = new DataOnMemoryListContainer<DataInstance>(dataUpdate.getAttributes());
        DataSet<DataOnMemory<DataInstance>> unionData = null;

        unionData =
                dataUpdate.getBatchedDataSet(this.batchSize)
                        .union(env.fromCollection(Arrays.asList(emtpyBatch),
                                TypeExtractor.getForClass((Class<DataOnMemory<DataInstance>>) Class.forName("eu.amidst.core.datastream.DataOnMemory"))));

        DataSet<CompoundVector> newparamSet =
                unionData
                .map(new ParallelVBMap(randomStart, idenitifableModelling))
                .withParameters(config)
                .withBroadcastSet(loop, "VB_PARAMS_" + this.dag.getName())
                .reduce(new ParallelVBReduce());

        // feed new centroids back into next iteration
        DataSet<CompoundVector> finlparamSet = loop.closeWith(newparamSet);

        parameterPrior = finlparamSet.collect().get(0);

        this.svb.updateNaturalParameterPosteriors(parameterPrior);

        this.svb.updateNaturalParameterPrior(parameterPrior);

        if(timeLimit == -1)
            this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO();
        else
            this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();

        this.svb.applyTransition();

    }catch(Exception ex){
        throw new RuntimeException(ex.getMessage());
    }

    this.randomStart=false;

    return this.getLogMarginalProbability();
}
Example 20
Source File: InstantiationUtil.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public static void writeObjectToConfig(Object o, Configuration config, String key) throws IOException {
	byte[] bytes = serializeObject(o);
	config.setBytes(key, bytes);
}