Java Code Examples for org.nd4j.linalg.dataset.api.DataSet#getFeatures()

@Test
public void testRPTree() throws Exception {
    DataSetIterator mnist = new MnistDataSetIterator(150,150);
    RPTree rpTree = new RPTree(784,50);
    DataSet d = mnist.next();
    NormalizerStandardize normalizerStandardize = new NormalizerStandardize();
    normalizerStandardize.fit(d);
    normalizerStandardize.transform(d.getFeatures());
    INDArray data = d.getFeatures();
    rpTree.buildTree(data);
    assertEquals(4,rpTree.getLeaves().size());
    assertEquals(0,rpTree.getRoot().getDepth());

    List<Integer> candidates = rpTree.getCandidates(data.getRow(0));
    assertFalse(candidates.isEmpty());
    assertEquals(10,rpTree.query(data.slice(0),10).length());
    System.out.println(candidates.size());

    rpTree.addNodeAtIndex(150,data.getRow(0));

}
 

@Override
public void preProcess(DataSet toPreProcess) {
    INDArray input = toPreProcess.getFeatures();
    if (input.rank() == 2)
        return; //No op: should usually never happen in a properly configured data pipeline

    //Assume input is standard rank 4 activations - i.e., CNN image data
    //First: we require input to be in c order. But c order (as declared in array order) isn't enough; also need strides to be correct
    if (input.ordering() != 'c' || !Shape.strideDescendingCAscendingF(input))
        input = input.dup('c');

    val inShape = input.shape(); //[miniBatch,depthOut,outH,outW]
    val outShape = new long[] {inShape[0], inShape[1] * inShape[2] * inShape[3]};

    INDArray reshaped = input.reshape('c', outShape);
    toPreProcess.setFeatures(reshaped);
}
 

/** Convert a DataSet to the equivalent MultiDataSet */
public static MultiDataSet toMultiDataSet(DataSet dataSet) {
    INDArray f = dataSet.getFeatures();
    INDArray l = dataSet.getLabels();
    INDArray fMask = dataSet.getFeaturesMaskArray();
    INDArray lMask = dataSet.getLabelsMaskArray();
    List<Serializable> meta = dataSet.getExampleMetaData();

    INDArray[] fNew = f == null ? null : new INDArray[] {f};
    INDArray[] lNew = l == null ? null : new INDArray[] {l};
    INDArray[] fMaskNew = (fMask != null ? new INDArray[] {fMask} : null);
    INDArray[] lMaskNew = (lMask != null ? new INDArray[] {lMask} : null);

    org.nd4j.linalg.dataset.MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(fNew, lNew, fMaskNew, lMaskNew);
    mds.setExampleMetaData(meta);
    return mds;
}
 

@Override
public void preProcess(DataSet toPreProcess) {
    INDArray input = toPreProcess.getFeatures();
    if (input.rank() == 2)
        return; //No op: should usually never happen in a properly configured data pipeline

    //Assume input is standard rank 4 activations - i.e., CNN image data
    //First: we require input to be in c order. But c order (as declared in array order) isn't enough; also need strides to be correct
    if (input.ordering() != 'c' || !Shape.strideDescendingCAscendingF(input))
        input = input.dup('c');

    val inShape = input.shape(); //[miniBatch,depthOut,outH,outW]
    val outShape = new long[] {inShape[0], inShape[1] * inShape[2] * inShape[3]};

    INDArray reshaped = input.reshape('c', outShape);
    toPreProcess.setFeatures(reshaped);
}
 

@Override
public void call(DataSet dataSet) {
    if (dataSet != null) {
        if (dataSet.getFeatures() != null)
            Nd4j.getAffinityManager().ensureLocation(dataSet.getFeatures(), AffinityManager.Location.DEVICE);

        if (dataSet.getLabels() != null)
            Nd4j.getAffinityManager().ensureLocation(dataSet.getLabels(), AffinityManager.Location.DEVICE);

        if (dataSet.getFeaturesMaskArray() != null)
            Nd4j.getAffinityManager().ensureLocation(dataSet.getFeaturesMaskArray(),
                            AffinityManager.Location.DEVICE);

        if (dataSet.getLabelsMaskArray() != null)
            Nd4j.getAffinityManager().ensureLocation(dataSet.getLabelsMaskArray(), AffinityManager.Location.DEVICE);
    }
}
 

/**
 * NOTE: The data format must be NHWC
 */
@Override
public void preProcess(DataSet dataSet) {
    Preconditions.checkNotNull(dataSet, "Encountered null dataSet");

    if(dataSet.isEmpty()) {
        return;
    }

    INDArray input = dataSet.getFeatures();
    INDArray output = Nd4j.create(LongShapeDescriptor.fromShape(resizedShape, input.dataType()), false);

    CustomOp op = DynamicCustomOp.builder("crop_and_resize")
            .addInputs(input, boxes, indices, resize)
            .addIntegerArguments(method)
            .addOutputs(output)
            .build();
    Nd4j.getExecutioner().exec(op);

    dataSet.setFeatures(output);
}
 

@Override
public void preProcess(DataSet dataSet) {
    Preconditions.checkNotNull(dataSet, "Encountered null dataSet");

    if(dataSet.isEmpty()) {
        return;
    }

    INDArray input = dataSet.getFeatures();
    INDArray output;
    switch (permutationType) {
        case NCHWtoNHWC:
            output = input.permute(0, 2, 3, 1);
            break;

        case NHWCtoNCHW:
            output = input.permute(0, 3, 1, 2);
            break;

        case Custom:
            output = input.permute(rearrange);
            break;

        default:
            output = input;
            break;
    }

    dataSet.setFeatures(output);
}
 

@Test
    public void testRocMultiToHtml() throws Exception {
        DataSetIterator iter = new IrisDataSetIterator(150, 150);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list()
                        .layer(0, new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()).layer(1,
                                        new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                        .build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        NormalizerStandardize ns = new NormalizerStandardize();
        DataSet ds = iter.next();
        ns.fit(ds);
        ns.transform(ds);

        for (int i = 0; i < 30; i++) {
            net.fit(ds);
        }

        for (int numSteps : new int[] {20, 0}) {
            ROCMultiClass roc = new ROCMultiClass(numSteps);
            iter.reset();

            INDArray f = ds.getFeatures();
            INDArray l = ds.getLabels();
            INDArray out = net.output(f);
            roc.eval(l, out);


            String str = EvaluationTools.rocChartToHtml(roc, Arrays.asList("setosa", "versicolor", "virginica"));
//            System.out.println(str);
        }
    }
 

@Override
public void preProcess(DataSet toPreProcess) {
    INDArray features = toPreProcess.getFeatures();
    preProcess(features);
    if(fitLabels && toPreProcess.getLabels() != null){
        preProcess(toPreProcess.getLabels());
    }
}
 

@Override
public void preProcess(DataSet dataSet) {
    Preconditions.checkNotNull(dataSet, "Encountered null dataSet");

    if(dataSet.isEmpty()) {
        return;
    }

    INDArray originalFeatures = dataSet.getFeatures();
    long[] originalShape = originalFeatures.shape();

    // result shape is NHW
    INDArray result = Nd4j.create(originalShape[0], originalShape[2], originalShape[3]);

    for(long n = 0, numExamples = originalShape[0]; n < numExamples; ++n) {
        // Extract channels
        INDArray itemFeatures = originalFeatures.slice(n, 0); // shape is CHW
        INDArray R = itemFeatures.slice(0, 0);  // shape is HW
        INDArray G = itemFeatures.slice(1, 0);
        INDArray B = itemFeatures.slice(2, 0);

        // Convert
        R.muli(RED_RATIO);
        G.muli(GREEN_RATIO);
        B.muli(BLUE_RATIO);
        R.addi(G).addi(B);

        result.putSlice((int)n, R);
    }

    dataSet.setFeatures(result);
}
 

@Override
public void preProcess(DataSet toPreProcess) {
    INDArray features = toPreProcess.getFeatures();
    this.preProcess(features);
}
 

@Override
public void preProcess(DataSet toPreProcess) {
    INDArray features = toPreProcess.getFeatures();
    this.preProcess(features);
}
 

@Override
public void preProcess(DataSet toPreProcess) {
    INDArray features = toPreProcess.getFeatures();
    this.preProcess(features);
}
 

@Test
public void testCnnSentenceDataSetIteratorUseUnknownVector() throws Exception {

    WordVectors w2v = WordVectorSerializer
            .readWord2VecModel(new ClassPathResource("word2vec/googleload/sample_vec.bin").getFile());

    List<String> sentences = new ArrayList<>();
    sentences.add("these balance Database model");
    sentences.add("into same THISWORDDOESNTEXIST are");
    //Last 2 sentences - no valid words
    sentences.add("NOVALID WORDSHERE");
    sentences.add("!!!");

    List<String> labelsForSentences = Arrays.asList("Positive", "Negative", "Positive", "Negative");


    LabeledSentenceProvider p = new CollectionLabeledSentenceProvider(sentences, labelsForSentences, null);
    CnnSentenceDataSetIterator dsi = new CnnSentenceDataSetIterator.Builder(CnnSentenceDataSetIterator.Format.CNN1D)
            .unknownWordHandling(CnnSentenceDataSetIterator.UnknownWordHandling.UseUnknownVector)
            .sentenceProvider(p).wordVectors(w2v)
            .useNormalizedWordVectors(true)
            .maxSentenceLength(256).minibatchSize(4).sentencesAlongHeight(false).build();

    assertTrue(dsi.hasNext());
    DataSet ds = dsi.next();

    assertFalse(dsi.hasNext());

    INDArray f = ds.getFeatures();
    assertEquals(4, f.size(0));

    INDArray unknown = w2v.getWordVectorMatrix(w2v.getUNK());
    if(unknown == null)
        unknown = Nd4j.create(DataType.FLOAT, f.size(1));

    assertEquals(unknown, f.get(NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.point(0)));
    assertEquals(unknown, f.get(NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.point(1)));
    assertEquals(unknown.like(), f.get(NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.point(3)));

    assertEquals(unknown, f.get(NDArrayIndex.point(3), NDArrayIndex.all(), NDArrayIndex.point(0)));
    assertEquals(unknown.like(), f.get(NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.point(1)));

    //Sanity check on single sentence loading:
    INDArray allKnownWords = dsi.loadSingleSentence("these balance");
    INDArray withUnknown = dsi.loadSingleSentence("these NOVALID");
    INDArray allUnknown = dsi.loadSingleSentence("NOVALID AlsoNotInVocab");
    assertNotNull(allKnownWords);
    assertNotNull(withUnknown);
    assertNotNull(allUnknown);
}
 

@Test
public void testRocHtml() {

    DataSetIterator iter = new IrisDataSetIterator(150, 150);

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()).layer(1,
                                    new OutputLayer.Builder().nIn(4).nOut(2).activation(Activation.SOFTMAX)
                                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    NormalizerStandardize ns = new NormalizerStandardize();
    DataSet ds = iter.next();
    ns.fit(ds);
    ns.transform(ds);

    INDArray newLabels = Nd4j.create(150, 2);
    newLabels.getColumn(0).assign(ds.getLabels().getColumn(0));
    newLabels.getColumn(0).addi(ds.getLabels().getColumn(1));
    newLabels.getColumn(1).assign(ds.getLabels().getColumn(2));
    ds.setLabels(newLabels);

    for (int i = 0; i < 30; i++) {
        net.fit(ds);
    }

    for (int numSteps : new int[] {20, 0}) {
        ROC roc = new ROC(numSteps);
        iter.reset();

        INDArray f = ds.getFeatures();
        INDArray l = ds.getLabels();
        INDArray out = net.output(f);
        roc.eval(l, out);


        String str = EvaluationTools.rocChartToHtml(roc);
        //            System.out.println(str);
    }
}
 

@Override
public void preProcess(DataSet toPreProcess) {
    INDArray features = toPreProcess.getFeatures();
    this.preProcess(features);
}
 

@Override
public void preProcess(DataSet toPreProcess) {
    INDArray features = toPreProcess.getFeatures();
    this.preProcess(features);
}