Java Code Examples for org.nd4j.evaluation.classification.Evaluation#Metric

public Builder trainEvaluationMetrics(String name, int labelIdx, Evaluation.Metric... metrics){
    if(trainEvalMetrics == null){
        trainEvalMetrics = new LinkedHashMap<>();
    }
    Pair<String,Integer> p = new Pair<>(name, labelIdx);
    if(!trainEvalMetrics.containsKey(p)){
        trainEvalMetrics.put(p, new ArrayList<Evaluation.Metric>());
    }
    List<Evaluation.Metric> l = trainEvalMetrics.get(p);
    for(Evaluation.Metric m : metrics){
        if(!l.contains(m)){
            l.add(m);
        }
    }
    return this;
}
 

@Override
public ListenerResponse epochEnd(SameDiff sd, At at, LossCurve lossCurve, long epochTimeMillis) {

    //If any training evaluation, report it here:
    if(epochTrainEval != null){
        long time = System.currentTimeMillis();
        for(Map.Entry<Pair<String,Integer>,Evaluation> e : epochTrainEval.entrySet()){
            String n = "evaluation/" + e.getKey().getFirst();   //TODO what if user does same eval with multiple labels? Doesn't make sense... add validation to ensure this?

            List<Evaluation.Metric> l = trainEvalMetrics.get(e.getKey());
            for(Evaluation.Metric m : l) {
                String mName = n + "/train/" + m.toString().toLowerCase();
                if (!wroteEvalNames) {
                    if(!writer.registeredEventName(mName)) {    //Might have been registered if continuing training
                        writer.registerEventNameQuiet(mName);
                    }
                }

                double score = e.getValue().scoreForMetric(m);
                try{
                    writer.writeScalarEvent(mName, LogFileWriter.EventSubtype.EVALUATION, time, at.iteration(), at.epoch(), score);
                } catch (IOException ex){
                    throw new RuntimeException("Error writing to log file", ex);
                }
            }

            wroteEvalNames = true;
        }
    }

    epochTrainEval = null;
    return ListenerResponse.CONTINUE;
}
 

@Test
public void testClassificationScoreFunctionSimple() throws Exception {

    for(Evaluation.Metric metric : Evaluation.Metric.values()) {
        log.info("Metric: " + metric);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .list()
                .layer(new DenseLayer.Builder().nIn(784).nOut(32).build())
                .layer(new OutputLayer.Builder().nIn(32).nOut(10).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        DataSetIterator iter = new MnistDataSetIterator(32, false, 12345);

        List<DataSet> l = new ArrayList<>();
        for( int i=0; i<10; i++ ){
            DataSet ds = iter.next();
            l.add(ds);
        }

        iter = new ExistingDataSetIterator(l);

        EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
        EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                        .epochTerminationConditions(new MaxEpochsTerminationCondition(5))
                        .iterationTerminationConditions(
                                new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES))
                        .scoreCalculator(new ClassificationScoreCalculator(metric, iter)).modelSaver(saver)
                        .build();

        EarlyStoppingTrainer trainer = new EarlyStoppingTrainer(esConf, net, iter);
        EarlyStoppingResult<MultiLayerNetwork> result = trainer.fit();

        assertNotNull(result.getBestModel());
    }
}
 

public ClassificationScoreCalculator(Evaluation.Metric metric, DataSetIterator iterator){
    super(iterator);
    this.metric = metric;
}
 

public ClassificationScoreCalculator(Evaluation.Metric metric, MultiDataSetIterator iterator){
    super(iterator);
    this.metric = metric;
}
 

@Test
public void testClassificationScoreFunctionSimple() throws Exception {

    for(Evaluation.Metric metric : Evaluation.Metric.values()) {
        log.info("Metric: " + metric);

        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                .graphBuilder()
                .addInputs("in")
                .layer("0", new DenseLayer.Builder().nIn(784).nOut(32).build(), "in")
                .layer("1", new OutputLayer.Builder().nIn(32).nOut(10).activation(Activation.SOFTMAX).build(), "0")
                .setOutputs("1")
                .build();

        ComputationGraph net = new ComputationGraph(conf);
        net.init();

        DataSetIterator iter = new MnistDataSetIterator(32, false, 12345);

        List<DataSet> l = new ArrayList<>();
        for( int i=0; i<10; i++ ){
            DataSet ds = iter.next();
            l.add(ds);
        }

        iter = new ExistingDataSetIterator(l);

        EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
        EarlyStoppingConfiguration<ComputationGraph> esConf =
                new EarlyStoppingConfiguration.Builder<ComputationGraph>()
                        .epochTerminationConditions(new MaxEpochsTerminationCondition(5))
                        .iterationTerminationConditions(
                                new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES))
                        .scoreCalculator(new ClassificationScoreCalculator(metric, iter)).modelSaver(saver)
                        .build();

        EarlyStoppingGraphTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, iter);
        EarlyStoppingResult<ComputationGraph> result = trainer.fit();

        assertNotNull(result.getBestModel());
    }
}
 

/**
 * @param metric Evaluation metric to calculate
 */
public EvaluationScoreFunction(@NonNull Evaluation.Metric metric) {
    this.metric = metric;
}