org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction Java Examples

@Test
public void GeneticSearchCandidateGenerator_getCandidate_GeneticExceptionShouldMarkCandidateAsFailed() {

    ScoreFunction scoreFunction = new BraninFunction.BraninScoreFunction();

    //Define configuration:
    CandidateGenerator candidateGenerator =
                    new GeneticSearchCandidateGenerator.Builder(new BraninFunction.BraninSpace(), scoreFunction)
                                    .selectionOperator(new TestSelectionOperator()).build();

    TestTerminationCondition testTerminationCondition = new TestTerminationCondition();

    OptimizationConfiguration configuration = new OptimizationConfiguration.Builder()
                    .candidateGenerator(candidateGenerator).scoreFunction(scoreFunction)
                    .terminationConditions(testTerminationCondition).build();

    IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new BraninFunction.BraninTaskCreator());

    runner.addListeners(new LoggingStatusListener());
    runner.execute();

    Assert.assertTrue(testTerminationCondition.hasAFailedCandidate);
}
 

private  ScoreFunction scoreFunctionMultiLayerNetwork() {
    if(problemType.equals(CLASSIFICIATION)) {
        switch(score) {
            case ACCURACY: return ScoreFunctions.testSetAccuracy();
            case F1: return ScoreFunctions.testSetF1();

            default: throw new IllegalArgumentException("Score " + score + " not valid for type " + problemType);
        }
    }
    else if(problemType.equals(REGRESSION)) {
        switch(score) {
            case REGRESSION_SCORE: return ScoreFunctions.testSetRegression(RegressionValue.valueOf(score));
            default: throw new IllegalArgumentException("Score " + score + " not valid for type " + problemType);

        }
    }
    throw new IllegalStateException("Illegal problem type " + problemType);
}
 

private  ScoreFunction scoreFunctionCompGraph() {
    if(problemType.equals(CLASSIFICIATION)) {
        switch(score) {
            case ACCURACY: return ScoreFunctions.testSetAccuracy();
            case F1: return ScoreFunctions.testSetF1();
            case F1_MULTI : return ScoreFunctions.testSetF1();
            case ACCURACY_MULTI: return ScoreFunctions.testSetAccuracy();

            default: throw new IllegalArgumentException("Score " + score + " not valid for type " + problemType);
        }
    }
    else if(problemType.equals(REGRESSION)) {
        switch(score) {
            case REGRESSION_SCORE: return ScoreFunctions.testSetRegression(RegressionValue.valueOf(score));
            case REGRESSION_SCORE_MULTI: return ScoreFunctions.testSetRegression(RegressionValue.valueOf(score));
            default: throw new IllegalArgumentException("Score " + score + " not valid for type " + problemType);
        }
    }
    throw new IllegalStateException("Illegal problem type " + problemType);
}
 

@Test
public void GeneticSearchCandidateGenerator_getCandidate_ShouldGenerateCandidates() throws Exception {

    ScoreFunction scoreFunction = new BraninFunction.BraninScoreFunction();

    //Define configuration:
    CandidateGenerator candidateGenerator =
                    new GeneticSearchCandidateGenerator.Builder(new BraninFunction.BraninSpace(), scoreFunction)
                                    .build();

    TestTerminationCondition testTerminationCondition = new TestTerminationCondition();
    OptimizationConfiguration configuration = new OptimizationConfiguration.Builder()
                    .candidateGenerator(candidateGenerator).scoreFunction(scoreFunction)
                    .terminationConditions(new MaxCandidatesCondition(50), testTerminationCondition).build();

    IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new BraninFunction.BraninTaskCreator());

    runner.addListeners(new LoggingStatusListener());
    runner.execute();

    Assert.assertFalse(testTerminationCondition.hasAFailedCandidate);
}
 

@Override
        public Callable<OptimizationResult> create(final Candidate c, DataProvider dataProvider,
                        final ScoreFunction scoreFunction, final List<StatusListener> statusListeners,
                        IOptimizationRunner runner) {

            return new Callable<OptimizationResult>() {
                @Override
                public OptimizationResult call() throws Exception {

                    BraninConfig candidate = (BraninConfig) c.getValue();

                    double score = scoreFunction.score(candidate, null, (Map) null);
//                    System.out.println(candidate.getX1() + "\t" + candidate.getX2() + "\t" + score);

                    Thread.sleep(20);

                    if (statusListeners != null) {
                        for (StatusListener sl : statusListeners) {
                            sl.onCandidateIteration(null, null, 0);
                        }
                    }

                    CandidateInfo ci = new CandidateInfo(-1, CandidateStatus.Complete, score,
                                    System.currentTimeMillis(), null, null, null, null);

                    return new OptimizationResult(c, score, c.getIndex(), null, ci, null);
                }
            };
        }
 

@Test
public void testScoreFunctionJson() throws Exception {

    ScoreFunction[] scoreFunctions = new ScoreFunction[]{
            ScoreFunctions.testSetAccuracy(), ScoreFunctions.testSetF1(),
            ScoreFunctions.testSetLoss(true), ScoreFunctions.testSetRegression(RegressionValue.MAE),
            ScoreFunctions.testSetRegression(RegressionValue.RMSE)};

    for(ScoreFunction sc : scoreFunctions){
        String json = JsonMapper.getMapper().writeValueAsString(sc);
        ScoreFunction fromJson = JsonMapper.getMapper().readValue(json, ScoreFunction.class);

        assertEquals(sc, fromJson);
    }
}
 

@Override
protected List<ListenableFuture<OptimizationResult>> execute(List<Candidate> candidates, DataProvider dataProvider,
                ScoreFunction scoreFunction) {
    List<ListenableFuture<OptimizationResult>> list = new ArrayList<>(candidates.size());
    for (Candidate candidate : candidates) {
        Callable<OptimizationResult> task =
                        taskCreator.create(candidate, dataProvider, scoreFunction, statusListeners, this);
        list.add(executor.submit(task));
    }
    return list;
}
 

public GraphLearningTask(Candidate candidate, Class<? extends DataSource> dataSource, Properties dataSourceProperties,
                         ScoreFunction scoreFunction, ModelEvaluator modelEvaluator, List<StatusListener> listeners,
                         TaskListener taskListener, IOptimizationRunner runner) {
    this.candidate = candidate;
    this.dataSource = dataSource;
    this.dataSourceProperties = dataSourceProperties;
    this.scoreFunction = scoreFunction;
    this.modelEvaluator = modelEvaluator;
    this.listeners = listeners;
    this.taskListener = taskListener;
    this.runner = runner;
}
 

public GraphLearningTask(Candidate candidate, DataProvider dataProvider, ScoreFunction scoreFunction,
                         ModelEvaluator modelEvaluator, List<StatusListener> listeners,
                         TaskListener taskListener, IOptimizationRunner runner) {
    this.candidate = candidate;
    this.dataProvider = dataProvider;
    this.scoreFunction = scoreFunction;
    this.modelEvaluator = modelEvaluator;
    this.listeners = listeners;
    this.taskListener = taskListener;
    this.runner = runner;
}
 

@Override
public Callable<OptimizationResult> create(Candidate candidate, DataProvider dataProvider,
                                           ScoreFunction scoreFunction, List<StatusListener> statusListener,
                                           IOptimizationRunner runner) {

    return new GraphLearningTask(candidate, dataProvider, scoreFunction, modelEvaluator, statusListener,
            taskListener, runner);
}
 

public DL4JLearningTask(Candidate candidate, Class<? extends DataSource> dataSource, Properties dataSourceProperties,
                        ScoreFunction scoreFunction, ModelEvaluator modelEvaluator, List<StatusListener> listeners, TaskListener taskListener,
                        IOptimizationRunner runner) {
    this.candidate = candidate;
    this.dataSource = dataSource;
    this.dataSourceProperties = dataSourceProperties;
    this.scoreFunction = scoreFunction;
    this.modelEvaluator = modelEvaluator;
    this.listeners = listeners;
    this.taskListener = taskListener;
    this.runner = runner;
}
 

public DL4JLearningTask(Candidate candidate, DataProvider dataProvider, ScoreFunction scoreFunction,
                ModelEvaluator modelEvaluator, List<StatusListener> listeners, TaskListener taskListener,
                        IOptimizationRunner runner) {
    this.candidate = candidate;
    this.dataProvider = dataProvider;
    this.scoreFunction = scoreFunction;
    this.modelEvaluator = modelEvaluator;
    this.listeners = listeners;
    this.taskListener = taskListener;
    this.runner = runner;
}
 

@Override
protected List<ListenableFuture<OptimizationResult>> execute(List<Candidate> candidates, Class<? extends DataSource> dataSource, Properties dataSourceProperties, ScoreFunction scoreFunction) {
    List<ListenableFuture<OptimizationResult>> list = new ArrayList<>(candidates.size());
    for (Candidate candidate : candidates) {
        Callable<OptimizationResult> task = taskCreator.create(candidate, dataSource, dataSourceProperties, scoreFunction, statusListeners, this);
        list.add(executor.submit(task));
    }
    return list;
}
 

@Override
public Callable<OptimizationResult> create(Candidate candidate, DataProvider dataProvider,
                                           ScoreFunction scoreFunction, List<StatusListener> statusListeners,
                                           IOptimizationRunner runner) {

    return new DL4JLearningTask(candidate, dataProvider, scoreFunction, modelEvaluator, statusListeners, taskListener, runner);
}
 

@Override
protected ListenableFuture<OptimizationResult> execute(Candidate candidate, Class<? extends DataSource> dataSource, Properties dataSourceProperties, ScoreFunction scoreFunction) {
    return execute(Collections.singletonList(candidate), dataSource, dataSourceProperties, scoreFunction).get(0);
}
 

@Deprecated
protected abstract ListenableFuture<OptimizationResult> execute(Candidate candidate, DataProvider dataProvider,
                                                                ScoreFunction scoreFunction);
 

@Deprecated
protected abstract List<ListenableFuture<OptimizationResult>> execute(List<Candidate> candidates,
                                                                      DataProvider dataProvider, ScoreFunction scoreFunction);
 

protected abstract ListenableFuture<OptimizationResult> execute(Candidate candidate, Class<? extends DataSource> dataSource,
Properties dataSourceProperties, ScoreFunction scoreFunction);
 

@Override
protected ListenableFuture<OptimizationResult> execute(Candidate candidate, DataProvider dataProvider,
                ScoreFunction scoreFunction) {
    return execute(Collections.singletonList(candidate), dataProvider, scoreFunction).get(0);
}
 

protected abstract List<ListenableFuture<OptimizationResult>> execute(List<Candidate> candidates, Class<? extends DataSource> dataSource,
Properties dataSourceProperties, ScoreFunction scoreFunction);
 

@Override
public Callable<OptimizationResult> create(Candidate candidate, Class<? extends DataSource> dataSource,
                Properties dataSourceProperties, ScoreFunction scoreFunction,
                List<StatusListener> statusListeners, IOptimizationRunner runner) {
    throw new UnsupportedOperationException();
}
 

@Test
public void testROCScoreFunctions() throws Exception {


    for (boolean auc : new boolean[]{true, false}) {
        for (ROCScoreFunction.ROCType rocType : ROCScoreFunction.ROCType.values()) {
            String msg = (auc ? "AUC" : "AUPRC") + " - " + rocType;
            log.info("Starting: " + msg);

            ParameterSpace<Double> lr = new ContinuousParameterSpace(1e-5, 1e-3);

            int nOut = (rocType == ROCScoreFunction.ROCType.ROC ? 2 : 10);
            LossFunctions.LossFunction lf = (rocType == ROCScoreFunction.ROCType.BINARY ?
                    LossFunctions.LossFunction.XENT : LossFunctions.LossFunction.MCXENT);
            Activation a = (rocType == ROCScoreFunction.ROCType.BINARY ? Activation.SIGMOID : Activation.SOFTMAX);
            MultiLayerSpace mls = new MultiLayerSpace.Builder()
                    .trainingWorkspaceMode(WorkspaceMode.NONE)
                    .inferenceWorkspaceMode(WorkspaceMode.NONE)
                    .updater(new AdamSpace(lr))
                    .weightInit(WeightInit.XAVIER)
                    .layer(new OutputLayerSpace.Builder().nIn(784).nOut(nOut)
                            .activation(a)
                            .lossFunction(lf).build())
                    .build();

            CandidateGenerator cg = new RandomSearchGenerator(mls);
            ResultSaver rs = new InMemoryResultSaver();
            ScoreFunction sf = new ROCScoreFunction(rocType, (auc ? ROCScoreFunction.Metric.AUC : ROCScoreFunction.Metric.AUPRC));


            OptimizationConfiguration oc = new OptimizationConfiguration.Builder()
                    .candidateGenerator(cg)
                    .dataProvider(new DP(rocType))
                    .modelSaver(rs)
                    .scoreFunction(sf)
                    .terminationConditions(new MaxCandidatesCondition(3))
                    .rngSeed(12345)
                    .build();

            IOptimizationRunner runner = new LocalOptimizationRunner(oc, new MultiLayerNetworkTaskCreator());
            runner.execute();

            List<ResultReference> list = runner.getResults();

            for (ResultReference rr : list) {
                DataSetIterator testIter = new MnistDataSetIterator(4, 16, false, false, false, 12345);
                testIter.setPreProcessor(new PreProc(rocType));

                OptimizationResult or = rr.getResult();
                MultiLayerNetwork net = (MultiLayerNetwork) or.getResultReference().getResultModel();

                double expScore;
                switch (rocType){
                    case ROC:
                        if(auc){
                            expScore = net.doEvaluation(testIter, new ROC())[0].calculateAUC();
                        } else {
                            expScore = net.doEvaluation(testIter, new ROC())[0].calculateAUCPR();
                        }
                        break;
                    case BINARY:
                        if(auc){
                            expScore = net.doEvaluation(testIter, new ROCBinary())[0].calculateAverageAuc();
                        } else {
                            expScore = net.doEvaluation(testIter, new ROCBinary())[0].calculateAverageAUCPR();
                        }
                        break;
                    case MULTICLASS:
                        if(auc){
                            expScore = net.doEvaluation(testIter, new ROCMultiClass())[0].calculateAverageAUC();
                        } else {
                            expScore = net.doEvaluation(testIter, new ROCMultiClass())[0].calculateAverageAUCPR();
                        }
                        break;
                    default:
                        throw new RuntimeException();
                }


                DataSetIterator iter = new MnistDataSetIterator(4, 16, false, false, false, 12345);
                iter.setPreProcessor(new PreProc(rocType));

                assertEquals(msg, expScore, or.getScore(), 1e-4);
            }
        }
    }
}
 

/**
 * @param parameterSpace ParameterSpace from which to generate candidates
 * @param scoreFunction The score function that will be used in the OptimizationConfiguration
 */
public Builder(ParameterSpace<?> parameterSpace, ScoreFunction scoreFunction) {
    this.parameterSpace = parameterSpace;
    this.minimizeScore = scoreFunction.minimize();
}
 

public static void main(String[] args) {

        ParameterSpace<Double> learningRateParam = new ContinuousParameterSpace(0.0001,0.01);
        ParameterSpace<Integer> layerSizeParam = new IntegerParameterSpace(5,11);
        MultiLayerSpace hyperParamaterSpace = new MultiLayerSpace.Builder()
                .updater(new AdamSpace(learningRateParam))
                //  .weightInit(WeightInit.DISTRIBUTION).dist(new LogNormalDistribution())
                .addLayer(new DenseLayerSpace.Builder()
                        .activation(Activation.RELU)
                        .nIn(11)
                        .nOut(layerSizeParam)
                        .build())
                .addLayer(new DenseLayerSpace.Builder()
                        .activation(Activation.RELU)
                        .nIn(layerSizeParam)
                        .nOut(layerSizeParam)
                        .build())
                .addLayer(new OutputLayerSpace.Builder()
                        .activation(Activation.SIGMOID)
                        .lossFunction(LossFunctions.LossFunction.XENT)
                        .nOut(1)
                        .build())
                .build();

        Map<String,Object> dataParams = new HashMap<>();
        dataParams.put("batchSize",new Integer(10));

        Map<String,Object> commands = new HashMap<>();
        commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, HyperParameterTuningArbiterUiExample.ExampleDataSource.class.getCanonicalName());

        CandidateGenerator candidateGenerator = new RandomSearchGenerator(hyperParamaterSpace,dataParams);

        Properties dataSourceProperties = new Properties();
        dataSourceProperties.setProperty("minibatchSize", "64");

        ResultSaver modelSaver = new FileModelSaver("resources/");
        ScoreFunction scoreFunction = new EvaluationScoreFunction(org.deeplearning4j.eval.Evaluation.Metric.ACCURACY);


        TerminationCondition[] conditions = {
                new MaxTimeCondition(120, TimeUnit.MINUTES),
                new MaxCandidatesCondition(30)

        };

        OptimizationConfiguration optimizationConfiguration = new OptimizationConfiguration.Builder()
                .candidateGenerator(candidateGenerator)
                .dataSource(HyperParameterTuningArbiterUiExample.ExampleDataSource.class,dataSourceProperties)
                .modelSaver(modelSaver)
                .scoreFunction(scoreFunction)
                .terminationConditions(conditions)
                .build();

        IOptimizationRunner runner = new LocalOptimizationRunner(optimizationConfiguration,new MultiLayerNetworkTaskCreator());
        //Uncomment this if you want to store the model.
        StatsStorage ss = new FileStatsStorage(new File("HyperParamOptimizationStats.dl4j"));
        runner.addListeners(new ArbiterStatusListener(ss));
        UIServer.getInstance().attach(ss);
        //runner.addListeners(new LoggingStatusListener()); //new ArbiterStatusListener(ss)
        runner.execute();

        //Print the best hyper params

        double bestScore = runner.bestScore();
        int bestCandidateIndex = runner.bestScoreCandidateIndex();
        int numberOfConfigsEvaluated = runner.numCandidatesCompleted();

        String s = "Best score: " + bestScore + "\n" +
                "Index of model with best score: " + bestCandidateIndex + "\n" +
                "Number of configurations evaluated: " + numberOfConfigsEvaluated + "\n";

        System.out.println(s);

    }
 

@Test
public void testInputTypeBasic() throws Exception {

    ParameterSpace<Integer> layerSizeHyperparam = new IntegerParameterSpace(20, 60);

    MultiLayerSpace hyperparameterSpace = new MultiLayerSpace.Builder().l2(0.0001)
                    .weightInit(WeightInit.XAVIER).updater(new Nesterovs())
                    .addLayer(new ConvolutionLayerSpace.Builder().kernelSize(5, 5).nIn(1).stride(1, 1)
                                    .nOut(layerSizeHyperparam).activation(Activation.IDENTITY).build())
                    .addLayer(new SubsamplingLayerSpace.Builder().poolingType(SubsamplingLayer.PoolingType.MAX)
                                    .kernelSize(2, 2).stride(2, 2).build())
                    .addLayer(new ConvolutionLayerSpace.Builder().kernelSize(5, 5)
                                    //Note that nIn need not be specified in later layers
                                    .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
                    .addLayer(new SubsamplingLayerSpace.Builder().poolingType(SubsamplingLayer.PoolingType.MAX)
                                    .kernelSize(2, 2).stride(2, 2).build())
                    .addLayer(new DenseLayerSpace.Builder().activation(Activation.RELU).nOut(500).build())
                    .addLayer(new OutputLayerSpace.Builder()
                                    .lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(10)
                                    .activation(Activation.SOFTMAX).build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();


    DataProvider dataProvider = new TestDataSetProvider();

    File f = testDir.newFolder();
    if (f.exists())
        f.delete();
    f.mkdir();
    ResultSaver modelSaver = new FileModelSaver(f.getAbsolutePath());

    ScoreFunction scoreFunction = new TestSetAccuracyScoreFunction();

    int maxCandidates = 4;
    TerminationCondition[] terminationConditions;
    terminationConditions = new TerminationCondition[] {new MaxCandidatesCondition(maxCandidates)};

    //Given these configuration options, let's put them all together:
    OptimizationConfiguration configuration = new OptimizationConfiguration.Builder()
                    .candidateGenerator(new RandomSearchGenerator(hyperparameterSpace, null))
                    .dataProvider(dataProvider).modelSaver(modelSaver).scoreFunction(scoreFunction)
                    .terminationConditions(terminationConditions).build();

    IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator());
    runner.execute();

    assertEquals(maxCandidates, runner.getResults().size());
}
 

@Override
public Callable<OptimizationResult> create(Candidate candidate, Class<? extends DataSource> dataSource, Properties dataSourceProperties,
                                           ScoreFunction scoreFunction, List<StatusListener> statusListeners, IOptimizationRunner runner) {
    return new GraphLearningTask(candidate, dataSource, dataSourceProperties, scoreFunction, modelEvaluator, statusListeners,
            taskListener, runner);
}
 

@Override
public Callable<OptimizationResult> create(Candidate candidate, Class<? extends DataSource> dataSource, Properties dataSourceProperties,
                                           ScoreFunction scoreFunction, List<StatusListener> statusListeners, IOptimizationRunner runner) {
    return new DL4JLearningTask(candidate, dataSource, dataSourceProperties, scoreFunction, modelEvaluator, statusListeners, taskListener, runner);
}
 

/**
 * Calculate a regression value (MSE, MAE etc) on a test set
 */
public static ScoreFunction testSetRegression(RegressionValue regressionValue) {
    return new TestSetRegressionScoreFunction(regressionValue);
}
 

/**
 * Calculate the f1 score on a test set
 */
public static ScoreFunction testSetF1() {
    return new TestSetF1ScoreFunction();
}
 

/**
 * Calculate the accuracy on a test set, for a MultiLayerNetwork
 */
public static ScoreFunction testSetAccuracy() {
    return new TestSetAccuracyScoreFunction();
}