Java Code Examples for org.nd4j.linalg.dataset.api.MultiDataSet#getFeaturesMaskArrays()

The following examples show how to use org.nd4j.linalg.dataset.api.MultiDataSet#getFeaturesMaskArrays() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: DefaultCallback.java From deeplearning4j with Apache License 2.0

6 votes

@Override
public void call(MultiDataSet multiDataSet) {
    if (multiDataSet != null) {
        if (multiDataSet.getFeatures() != null)
            for (int i = 0; i < multiDataSet.getFeatures().length; i++)
                Nd4j.getAffinityManager().ensureLocation(multiDataSet.getFeatures()[i],
                                AffinityManager.Location.DEVICE);

        if (multiDataSet.getLabels() != null)
            for (int i = 0; i < multiDataSet.getLabels().length; i++)
                Nd4j.getAffinityManager().ensureLocation(multiDataSet.getLabels()[i],
                                AffinityManager.Location.DEVICE);

        if (multiDataSet.getFeaturesMaskArrays() != null)
            for (int i = 0; i < multiDataSet.getFeaturesMaskArrays().length; i++)
                Nd4j.getAffinityManager().ensureLocation(multiDataSet.getFeaturesMaskArrays()[i],
                                AffinityManager.Location.DEVICE);

        if (multiDataSet.getLabelsMaskArrays() != null)
            for (int i = 0; i < multiDataSet.getLabelsMaskArrays().length; i++)
                Nd4j.getAffinityManager().ensureLocation(multiDataSet.getLabelsMaskArrays()[i],
                                AffinityManager.Location.DEVICE);
    }
}

Example 2

Source File: MultiLayerNetwork.java From deeplearning4j with Apache License 2.0

6 votes

@Override
public void fit(MultiDataSet dataSet) {
    if (dataSet.getFeatures().length == 1 && dataSet.getLabels().length == 1) {
        INDArray features = dataSet.getFeatures(0);
        INDArray labels = dataSet.getLabels(0);
        INDArray fMask = null;
        INDArray lMask = null;

        if (dataSet.getFeaturesMaskArrays() != null)
            fMask = dataSet.getFeaturesMaskArrays()[0];

        if (dataSet.getFeaturesMaskArrays() != null)
            lMask = dataSet.getLabelsMaskArrays()[0];

        DataSet ds = new DataSet(features, labels, fMask, lMask);
        fit(ds);
    } else {
        throw new DL4JInvalidInputException(
                "MultiLayerNetwork can't handle MultiDataSet with more than 1 features or labels array." +
                        "Please consider use of ComputationGraph");
    }
}

Example 3

Source File: BertIterator.java From deeplearning4j with Apache License 2.0

6 votes

/**
 * For use during inference. Will convert a given list of sentences to features and feature masks as appropriate.
 *
 * @param listOnlySentences
 * @return Pair of INDArrays[], first element is feature arrays and the second is the masks array
 */
public Pair<INDArray[], INDArray[]> featurizeSentences(List<String> listOnlySentences) {

    List<Pair<String, String>> sentencesWithNullLabel = addDummyLabel(listOnlySentences);
    SentenceListProcessed sentenceListProcessed = tokenizeMiniBatch(sentencesWithNullLabel);
    List<Pair<List<String>, String>> tokensAndLabelList = sentenceListProcessed.getTokensAndLabelList();
    int outLength = sentenceListProcessed.getMaxL();

    if (preProcessor != null) {
        Pair<INDArray[], INDArray[]> featureFeatureMasks = convertMiniBatchFeatures(tokensAndLabelList, outLength, null);
        MultiDataSet dummyMDS = new org.nd4j.linalg.dataset.MultiDataSet(featureFeatureMasks.getFirst(), null, featureFeatureMasks.getSecond(), null);
        preProcessor.preProcess(dummyMDS);
        return new Pair<>(dummyMDS.getFeatures(), dummyMDS.getFeaturesMaskArrays());
    }
    return convertMiniBatchFeatures(tokensAndLabelList, outLength, null);
}

Example 4

Source File: BertIterator.java From deeplearning4j with Apache License 2.0

6 votes

/**
 * For use during inference. Will convert a given pair of a list of sentences to features and feature masks as appropriate.
 *
 * @param listOnlySentencePairs
 * @return Pair of INDArrays[], first element is feature arrays and the second is the masks array
 */
public Pair<INDArray[], INDArray[]> featurizeSentencePairs(List<Pair<String, String>> listOnlySentencePairs) {
    Preconditions.checkState(sentencePairProvider != null, "The featurizeSentencePairs method is meant for inference with sentence pairs. Use only when the sentence pair provider is set (i.e not null).");

    List<Triple<String, String, String>> sentencePairsWithNullLabel = addDummyLabelForPairs(listOnlySentencePairs);
    SentencePairListProcessed sentencePairListProcessed = tokenizePairsMiniBatch(sentencePairsWithNullLabel);
    List<Pair<List<String>, String>> tokensAndLabelList = sentencePairListProcessed.getTokensAndLabelList();
    int outLength = sentencePairListProcessed.getMaxL();
    long[] segIdOnesFrom = sentencePairListProcessed.getSegIdOnesFrom();
    if (preProcessor != null) {
        Pair<INDArray[], INDArray[]> featuresAndMaskArraysPair = convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom);
        MultiDataSet dummyMDS = new org.nd4j.linalg.dataset.MultiDataSet(featuresAndMaskArraysPair.getFirst(), null, featuresAndMaskArraysPair.getSecond(), null);
        preProcessor.preProcess(dummyMDS);
        return new Pair<>(dummyMDS.getFeatures(), dummyMDS.getFeaturesMaskArrays());
    }
    return convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom);
}

Example 5

Source File: TestBertIterator.java From deeplearning4j with Apache License 2.0

6 votes

@Test
public void testSentencePairFeaturizer() throws IOException {
    int minibatchSize = 2;
    TestSentencePairsHelper testPairHelper = new TestSentencePairsHelper(minibatchSize);
    BertIterator b = BertIterator.builder()
            .tokenizer(testPairHelper.getTokenizer())
            .minibatchSize(minibatchSize)
            .padMinibatches(true)
            .featureArrays(BertIterator.FeatureArrays.INDICES_MASK_SEGMENTID)
            .vocabMap(testPairHelper.getTokenizer().getVocab())
            .task(BertIterator.Task.SEQ_CLASSIFICATION)
            .lengthHandling(BertIterator.LengthHandling.FIXED_LENGTH, 128)
            .sentencePairProvider(testPairHelper.getPairSentenceProvider())
            .prependToken("[CLS]")
            .appendToken("[SEP]")
            .build();
    MultiDataSet mds = b.next();
    INDArray[] featuresArr = mds.getFeatures();
    INDArray[] featuresMaskArr = mds.getFeaturesMaskArrays();

    Pair<INDArray[], INDArray[]> p = b.featurizeSentencePairs(testPairHelper.getSentencePairs());
    assertEquals(p.getFirst().length, 2);
    assertEquals(featuresArr[0], p.getFirst()[0]);
    assertEquals(featuresArr[1], p.getFirst()[1]);
    assertEquals(featuresMaskArr[0], p.getSecond()[0]);
}

Example 6

Source File: MultiDataSetWrapperIterator.java From deeplearning4j with Apache License 2.0

6 votes

@Override
public DataSet next() {
    MultiDataSet mds = iterator.next();
    if (mds.getFeatures().length > 1 || mds.getLabels().length > 1)
        throw new UnsupportedOperationException(
                        "This iterator is able to convert MultiDataSet with number of inputs/outputs of 1");

    INDArray features = mds.getFeatures()[0];
    INDArray labels = mds.getLabels() != null ? mds.getLabels()[0] : features;
    INDArray fMask = mds.getFeaturesMaskArrays() != null ? mds.getFeaturesMaskArrays()[0] : null;
    INDArray lMask = mds.getLabelsMaskArrays() != null ? mds.getLabelsMaskArrays()[0] : null;

    DataSet ds = new DataSet(features, labels, fMask, lMask);

    if (preProcessor != null)
        preProcessor.preProcess(ds);

    return ds;
}