org.nd4j.linalg.util.FeatureUtil Java Examples

@Test
public void testSplitTestAndTrain() throws Exception {
    INDArray labels = FeatureUtil.toOutcomeMatrix(new int[] {0, 0, 0, 0, 0, 0, 0, 0}, 1);
    DataSet data = new DataSet(Nd4j.rand(8, 1), labels);

    SplitTestAndTrain train = data.splitTestAndTrain(6, new Random(1));
    assertEquals(train.getTrain().getLabels().length(), 6);

    SplitTestAndTrain train2 = data.splitTestAndTrain(6, new Random(1));
    assertEquals(getFailureMessage(), train.getTrain().getFeatureMatrix(), train2.getTrain().getFeatureMatrix());

    DataSet x0 = new IrisDataSetIterator(150, 150).next();
    SplitTestAndTrain testAndTrain = x0.splitTestAndTrain(10);
    assertArrayEquals(new long[] {10, 4}, testAndTrain.getTrain().getFeatureMatrix().shape());
    assertEquals(x0.getFeatureMatrix().getRows(ArrayUtil.range(0, 10)), testAndTrain.getTrain().getFeatureMatrix());
    assertEquals(x0.getLabels().getRows(ArrayUtil.range(0, 10)), testAndTrain.getTrain().getLabels());


}
 

public Pair<INDArray, opencv_core.Mat> convertMat(byte[] byteFeature) {
    INDArray label = FeatureUtil.toOutcomeVector(byteFeature[0], NUM_LABELS);; // first value in the 3073 byte array
    opencv_core.Mat image = new opencv_core.Mat(HEIGHT, WIDTH, CV_8UC(CHANNELS)); // feature are 3072
    ByteBuffer imageData = image.createBuffer();

    for (int i = 0; i < HEIGHT * WIDTH; i++) {
        imageData.put(3 * i, byteFeature[i + 1 + 2 * HEIGHT * WIDTH]); // blue
        imageData.put(3 * i + 1, byteFeature[i + 1 + HEIGHT * WIDTH]); // green
        imageData.put(3 * i + 2, byteFeature[i + 1]); // red
    }
    //        if (useSpecialPreProcessCifar) {
    //            image = convertCifar(image);
    //        }

    return new Pair<>(label, image);
}
 

public Pair<INDArray, Mat> convertMat(byte[] byteFeature) {
    INDArray label = FeatureUtil.toOutcomeVector(byteFeature[0], NUM_LABELS);; // first value in the 3073 byte array
    Mat image = new Mat(HEIGHT, WIDTH, CV_8UC(CHANNELS)); // feature are 3072
    ByteBuffer imageData = image.createBuffer();

    for (int i = 0; i < HEIGHT * WIDTH; i++) {
        imageData.put(3 * i, byteFeature[i + 1 + 2 * HEIGHT * WIDTH]); // blue
        imageData.put(3 * i + 1, byteFeature[i + 1 + HEIGHT * WIDTH]); // green
        imageData.put(3 * i + 2, byteFeature[i + 1]); // red
    }
    //        if (useSpecialPreProcessCifar) {
    //            image = convertCifar(image);
    //        }

    return new Pair<>(label, image);
}
 

@Test
public void testStringListLabels() {
    INDArray trueOutcome = FeatureUtil.toOutcomeVector(0, 2);
    INDArray predictedOutcome = FeatureUtil.toOutcomeVector(0, 2);

    List<String> labelsList = new ArrayList<>();
    labelsList.add("hobbs");
    labelsList.add("cal");

    Evaluation eval = new Evaluation(labelsList);

    eval.eval(trueOutcome, predictedOutcome);
    assertEquals(1, eval.classCount(0));
    assertEquals(labelsList.get(0), eval.getClassLabel(0));

}
 

@Test
public void testStringHashLabels() {
    INDArray trueOutcome = FeatureUtil.toOutcomeVector(0, 2);
    INDArray predictedOutcome = FeatureUtil.toOutcomeVector(0, 2);

    Map<Integer, String> labelsMap = new HashMap<>();
    labelsMap.put(0, "hobbs");
    labelsMap.put(1, "cal");

    Evaluation eval = new Evaluation(labelsMap);

    eval.eval(trueOutcome, predictedOutcome);
    assertEquals(1, eval.classCount(0));
    assertEquals(labelsMap.get(0), eval.getClassLabel(0));

}
 

@Test
public void testSplitTestAndTrain() {
    INDArray labels = FeatureUtil.toOutcomeMatrix(new int[] {0, 0, 0, 0, 0, 0, 0, 0}, 1);
    DataSet data = new DataSet(Nd4j.rand(8, 1), labels);

    SplitTestAndTrain train = data.splitTestAndTrain(6, new Random(1));
    assertEquals(train.getTrain().getLabels().length(), 6);

    SplitTestAndTrain train2 = data.splitTestAndTrain(6, new Random(1));
    assertEquals(getFailureMessage(), train.getTrain().getFeatures(), train2.getTrain().getFeatures());

    DataSet x0 = new IrisDataSetIterator(150, 150).next();
    SplitTestAndTrain testAndTrain = x0.splitTestAndTrain(10);
    assertArrayEquals(new long[] {10, 4}, testAndTrain.getTrain().getFeatures().shape());
    assertEquals(x0.getFeatures().getRows(ArrayUtil.range(0, 10)), testAndTrain.getTrain().getFeatures());
    assertEquals(x0.getLabels().getRows(ArrayUtil.range(0, 10)), testAndTrain.getTrain().getLabels());


}
 

/**
 *
 * @param point
 * @param numPossibleLabels
 * @return {@link DataSet}
 */
private static DataSet fromLabeledPoint(LabeledPoint point, long numPossibleLabels) {
    Vector features = point.features();
    double label = point.label();

    // FIXMEL int cast
    double[] fArr = features.toArray();
    return new DataSet(Nd4j.create(fArr, new long[]{1,fArr.length}),
                    FeatureUtil.toOutcomeVector((int) label, (int) numPossibleLabels));
}
 

@Test
public void testDenseToOutputLayer() {
    Nd4j.getRandom().setSeed(12345);
    final int numRows = 76;
    final int numColumns = 76;
    int nChannels = 3;
    int outputNum = 6;
    int seed = 123;

    //setup the network
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                    .l1(1e-1).l2(2e-4).dropOut(0.5).miniBatch(true)
                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                    .layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                    .build())
                    .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                    .build())
                    .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
                    .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                    .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                    .build())

                    .setInputType(InputType.convolutional(numRows, numColumns, nChannels));

    DataSet d = new DataSet(Nd4j.rand(new int[]{10, nChannels, numRows, numColumns}),
                    FeatureUtil.toOutcomeMatrix(new int[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, 6));
    MultiLayerNetwork network = new MultiLayerNetwork(builder.build());
    network.init();
    network.fit(d);

}
 

@Override
public DataSet vectorize(String text, String label) {
    INDArray input = transform(text);
    INDArray labelMatrix = FeatureUtil.toOutcomeVector(labelsSource.indexOf(label), labelsSource.size());

    return new DataSet(input, labelMatrix);
}
 

/**
 * Vectorizes the passed in text treating it as one document
 *
 * @param text  the text to vectorize
 * @param label the label of the text
 * @return a dataset with a transform of weights(relative to impl; could be word counts or tfidf scores)
 */
@Override
public DataSet vectorize(String text, String label) {
    INDArray input = transform(text);
    INDArray labelMatrix = FeatureUtil.toOutcomeVector(labelsSource.indexOf(label), labelsSource.size());

    return new DataSet(input, labelMatrix);
}
 

public static INDArray toLabelMatrix(List<String> labels, List<Window> windows) {
    int columns = labels.size();
    INDArray ret = Nd4j.create(windows.size(), columns);
    for (int i = 0; i < ret.rows(); i++) {
        ret.putRow(i, FeatureUtil.toOutcomeVector(labels.indexOf(windows.get(i).getLabel()), labels.size()));
    }
    return ret;
}
 

private DataSet fromCache() {
    INDArray outcomes = null;
    INDArray input = null;
    input = Nd4j.create(batch, vec.lookupTable().layerSize() * vec.getWindow());
    outcomes = Nd4j.create(batch, labels.size());
    for (int i = 0; i < batch; i++) {
        input.putRow(i, WindowConverter.asExampleMatrix(cache.get(i), vec));
        int idx = labels.indexOf(cache.get(i).getLabel());
        if (idx < 0)
            idx = 0;
        outcomes.putRow(i, FeatureUtil.toOutcomeVector(idx, labels.size()));
    }
    return new DataSet(input, outcomes);

}
 

/**
 * Fit the model for one iteration on the provided data
 *
 * @param examples the examples to classify (one example in each row)
 * @param labels   the labels for each example (the number of labels must match
 */
@Override
public void fit(INDArray examples, int[] labels) {
    org.deeplearning4j.nn.conf.layers.OutputLayer layerConf =
            (org.deeplearning4j.nn.conf.layers.OutputLayer) getOutputLayer().conf().getLayer();

    if (layerConf.getNOut() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();
    fit(examples, FeatureUtil.toOutcomeMatrix(labels, (int) layerConf.getNOut()));
}
 

/**
 * Fit the model
 *
 * @param examples the examples to classify (one example in each row)
 * @param labels   the labels for each example (the number of labels must match
 */
@Override
public void fit(INDArray examples, int[] labels) {
    INDArray outcomeMatrix = FeatureUtil.toOutcomeMatrix(labels, numLabels());
    fit(examples, outcomeMatrix);

}
 

/**
 * Sets the outcome of a particular example
 *
 * @param example the example to transform
 * @param label   the label of the outcome
 */
@Override
public void setOutcome(int example, int label) {
    if (example > numExamples())
        throw new IllegalArgumentException("No example at " + example);
    if (label > numOutcomes() || label < 0)
        throw new IllegalArgumentException("Illegal label");

    INDArray outcome = FeatureUtil.toOutcomeVector(label, numOutcomes());
    getLabels().putRow(example, outcome);
}
 

@Test
public void testEval() {
    int classNum = 5;
    Evaluation eval = new Evaluation (classNum);

    // Testing the edge case when some classes do not have true positive
    INDArray trueOutcome = FeatureUtil.toOutcomeVector(0, 5); //[1,0,0,0,0]
    INDArray predictedOutcome = FeatureUtil.toOutcomeVector(0, 5); //[1,0,0,0,0]
    eval.eval(trueOutcome, predictedOutcome);
    assertEquals(1, eval.classCount(0));
    assertEquals(1.0, eval.f1(), 1e-1);

    // Testing more than one sample. eval() does not reset the Evaluation instance
    INDArray trueOutcome2 = FeatureUtil.toOutcomeVector(1, 5); //[0,1,0,0,0]
    INDArray predictedOutcome2 = FeatureUtil.toOutcomeVector(0, 5); //[1,0,0,0,0]
    eval.eval(trueOutcome2, predictedOutcome2);
    // Verified with sklearn in Python
    // from sklearn.metrics import classification_report
    // classification_report(['a', 'a'], ['a', 'b'], labels=['a', 'b', 'c', 'd', 'e'])
    assertEquals(eval.f1(), 0.6, 1e-1);
    // The first entry is 0 label
    assertEquals(1, eval.classCount(0));
    // The first entry is 1 label
    assertEquals(1, eval.classCount(1));
    // Class 0: one positive, one negative -> (one true positive, one false positive); no true/false negatives
    assertEquals(1, eval.positive().get(0), 0);
    assertEquals(1, eval.negative().get(0), 0);
    assertEquals(1, eval.truePositives().get(0), 0);
    assertEquals(1, eval.falsePositives().get(0), 0);
    assertEquals(0, eval.trueNegatives().get(0), 0);
    assertEquals(0, eval.falseNegatives().get(0), 0);


    // The rest are negative
    assertEquals(1, eval.negative().get(0), 0);
    // 2 rows and only the first is correct
    assertEquals(0.5, eval.accuracy(), 0);
}
 

public DataSet getDataFor(int i) {
    File image = new File(images.get(i));
    int outcome = outcomes.indexOf(image.getParentFile().getAbsolutePath());
    try {
        return new DataSet(loader.asRowVector(image), FeatureUtil.toOutcomeVector(outcome, outcomes.size()));
    } catch (Exception e) {
        throw new IllegalStateException("Unable to getFromOrigin data for image " + i + " for path " + images.get(i));
    }
}
 

/**
 * Sets the outcome of a particular example
 *
 * @param example the example to transform
 * @param label   the label of the outcome
 */
@Override
public void setOutcome(int example, int label) {
    if (example > numExamples())
        throw new IllegalArgumentException("No example at " + example);
    if (label > numOutcomes() || label < 0)
        throw new IllegalArgumentException("Illegal label");

    INDArray outcome = FeatureUtil.toOutcomeVector(label, numOutcomes());
    getLabels().putRow(example, outcome);
}
 

/**
 * Divides the input data transform
 * by the max number in each row
 */
@Override
public void scale() {
    FeatureUtil.scaleByMax(getFeatures());
}
 

@Override
public void scaleMinAndMax(double min, double max) {
    FeatureUtil.scaleMinMax(min, max, getFeatures());
}
 

public DataSet fromImageFile(int label,File image) throws Exception {
    INDArray outcome = FeatureUtil.toOutcomeVector(label, numNames);
    INDArray image2 = ArrayUtil.toNDArray(loader.flattenedImageFromFile(image));
    return new DataSet(image2,outcome);
}
 

/**
 * Divides the input data transform
 * by the max number in each row
 */
@Override
public void scale() {
    FeatureUtil.scaleByMax(getFeatures());
}
 

@Override
public void scaleMinAndMax(double min, double max) {
    FeatureUtil.scaleMinMax(min, max, getFeatureMatrix());
}
 

@Override
public Tuple2<Double, DataSet> call(Tuple2<Text, BytesWritable> inputTuple) throws Exception {
    int lenFeatureVector = 0;

    if (numPossibleLabels >= 1) {
        lenFeatureVector = byteFileLen - 1;
        if (labelIndex < 0)
            labelIndex = byteFileLen - 1;
    }

    InputStream inputStream = new DataInputStream(new ByteArrayInputStream(inputTuple._2().getBytes()));

    int batchNumCount = 0;
    byte[] byteFeature = new byte[byteFileLen];
    List<DataSet> dataSets = new ArrayList<>();
    INDArray label;
    int featureCount;

    try {
        INDArray featureVector = Nd4j.create(lenFeatureVector);
        while ((inputStream.read(byteFeature)) != -1 && batchNumCount != batchSize) {
            featureCount = 0;
            label = FeatureUtil.toOutcomeVector(byteFeature[labelIndex], numPossibleLabels);
            for (int j = 1; j <= featureVector.length(); j++)
                featureVector.putScalar(featureCount++, byteFeature[j]);
            dataSets.add(new DataSet(featureVector, label));
            batchNumCount++;
            byteFeature = new byte[byteFileLen];
            featureVector = Nd4j.create(lenFeatureVector);
        }
    } catch (IOException e) {
        log.error("",e);
    }

    List<INDArray> inputs = new ArrayList<>();
    List<INDArray> labels = new ArrayList<>();

    for (DataSet data : dataSets) {
        inputs.add(data.getFeatures());
        labels.add(data.getLabels());
    }

    DataSet ds = new DataSet(Nd4j.vstack(inputs.toArray(new INDArray[0])),
                    Nd4j.vstack(labels.toArray(new INDArray[0])));
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return new Tuple2<>((double) batchNumCount, ds);

}
 

@Override
public DataSet call(List<List<Writable>> input) throws Exception {
    Iterator<List<Writable>> iter = input.iterator();

    INDArray features = null;
    INDArray labels = Nd4j.zeros(1, (regression ? 1 : numPossibleLabels), input.size());

    int[] fIdx = new int[3];
    int[] lIdx = new int[3];

    int i = 0;
    while (iter.hasNext()) {
        List<Writable> step = iter.next();
        if (i == 0) {
            features = Nd4j.zeros(1, step.size() - 1, input.size());
        }

        Iterator<Writable> timeStepIter = step.iterator();
        int countIn = 0;
        int countFeatures = 0;
        while (timeStepIter.hasNext()) {
            Writable current = timeStepIter.next();
            if (converter != null)
                current = converter.convert(current);
            if (countIn++ == labelIndex) {
                //label
                if (regression) {
                    lIdx[2] = i;
                    labels.putScalar(lIdx, current.toDouble());
                } else {
                    INDArray line = FeatureUtil.toOutcomeVector(current.toInt(), numPossibleLabels);
                    labels.tensorAlongDimension(i, 1).assign(line); //1d from [1,nOut,timeSeriesLength] -> tensor i along dimension 1 is at time i
                }
            } else {
                //feature
                fIdx[1] = countFeatures++;
                fIdx[2] = i;
                try {
                    features.putScalar(fIdx, current.toDouble());
                } catch (UnsupportedOperationException e) {
                    // This isn't a scalar, so check if we got an array already
                    if (current instanceof NDArrayWritable) {
                        features.get(NDArrayIndex.point(fIdx[0]), NDArrayIndex.all(), NDArrayIndex.point(fIdx[2]))
                                        .putRow(0, ((NDArrayWritable) current).get());
                    } else {
                        throw e;
                    }
                }
            }
        }
        i++;
    }

    DataSet ds = new DataSet(features, labels);
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return ds;
}
 

/**
 * Creates an output label matrix
 *
 * @param outcomeLabel the outcome label to use
 * @return a binary vector where 1 is transform to the
 * index specified by outcomeLabel
 */
protected INDArray createOutputVector(int outcomeLabel) {
    return FeatureUtil.toOutcomeVector(outcomeLabel, numOutcomes);
}
 

/**
 * Creates an output label matrix
 *
 * @param outcomeLabel the outcome label to use
 * @return a binary vector where 1 is transform to the
 * index specified by outcomeLabel
 */
protected INDArray createOutputVector(int outcomeLabel) {
    return FeatureUtil.toOutcomeVector(outcomeLabel, numOutcomes);
}