Java Code Examples for org.nd4j.linalg.dataset.DataSet#load()

@Test
public void testDataSetSaveLost() throws Exception {
    INDArray features = Nd4j.linspace(1, 16 * 784, 16 * 784).reshape(16, 784);
    INDArray labels = Nd4j.linspace(1, 160, 160).reshape(16, 10);

    for (int i = 0; i < 100; i++) {
        DataSet ds = new DataSet(features, labels);

        File tempFile = File.createTempFile("dataset", "temp");
        tempFile.deleteOnExit();

        ds.save(tempFile);

        DataSet restore = new DataSet();
        restore.load(tempFile);

        assertEquals(features, restore.getFeatureMatrix());
        assertEquals(labels, restore.getLabels());

    }
}
 

@Test
public void endlessTestSerDe1() throws Exception {
    INDArray features = Nd4j.create(32, 3, 224, 224);
    INDArray labels = Nd4j.create(32, 200);
    File tmp = File.createTempFile("12dadsad", "dsdasds");
    float[] array = new float[33 * 3 * 224 * 224];
    DataSet ds = new DataSet(features, labels);
    ds.save(tmp);

    WorkspaceConfiguration wsConf = WorkspaceConfiguration.builder().initialSize(0)
                    .policyLearning(LearningPolicy.FIRST_LOOP).build();

    while (true) {

        try (MemoryWorkspace workspace = Nd4j.getWorkspaceManager().getAndActivateWorkspace(wsConf, "serde")) {
            /*
                        try (FileOutputStream fos = new FileOutputStream(tmp); BufferedOutputStream bos = new BufferedOutputStream(fos)) {
            SerializationUtils.serialize(array, fos);
                        }
            
                        try (FileInputStream fis = new FileInputStream(tmp); BufferedInputStream bis = new BufferedInputStream(fis)) {
            long time1 = System.currentTimeMillis();
            float[] arrayR = (float[]) SerializationUtils.deserialize(bis);
            long time2 = System.currentTimeMillis();
            
            log.info("Load time: {}", time2 - time1);
                        }
            */



            long time1 = System.currentTimeMillis();
            ds.load(tmp);
            long time2 = System.currentTimeMillis();

            log.info("Load time: {}", time2 - time1);
        }
    }
}
 

@Override
public DataSet get(String key) {
    File file = resolveKey(key);

    if (!file.exists()) {
        return null;
    } else if (!file.isFile()) {
        throw new IllegalStateException("ERROR: cannot read DataSet: cache path " + file + " is not a file");
    } else {
        DataSet ds = new DataSet();
        ds.load(file);
        return ds;
    }
}
 

@Test
public void endlessTestSerDe1() throws Exception {
    INDArray features = Nd4j.create(32, 3, 224, 224);
    INDArray labels = Nd4j.create(32, 200);
    File tmp = File.createTempFile("12dadsad", "dsdasds");
    float[] array = new float[33 * 3 * 224 * 224];
    DataSet ds = new DataSet(features, labels);
    ds.save(tmp);

    WorkspaceConfiguration wsConf = WorkspaceConfiguration.builder().initialSize(0)
                    .policyLearning(LearningPolicy.FIRST_LOOP).build();

    while (true) {

        try (MemoryWorkspace workspace = Nd4j.getWorkspaceManager().getAndActivateWorkspace(wsConf, "serde")) {
            /*
                        try (FileOutputStream fos = new FileOutputStream(tmp); BufferedOutputStream bos = new BufferedOutputStream(fos)) {
            SerializationUtils.serialize(array, fos);
                        }
            
                        try (FileInputStream fis = new FileInputStream(tmp); BufferedInputStream bis = new BufferedInputStream(fis)) {
            long time1 = System.currentTimeMillis();
            float[] arrayR = (float[]) SerializationUtils.deserialize(bis);
            long time2 = System.currentTimeMillis();
            
            log.info("Load time: {}", time2 - time1);
                        }
            */



            long time1 = System.currentTimeMillis();
            ds.load(tmp);
            long time2 = System.currentTimeMillis();

            log.info("Load time: {}", time2 - time1);
        }
    }
}
 

@Override
public DataSet get(String key) {
    File file = resolveKey(key);

    if (!file.exists()) {
        return null;
    } else if (!file.isFile()) {
        throw new IllegalStateException("ERROR: cannot read DataSet: cache path " + file + " is not a file");
    } else {
        DataSet ds = new DataSet();
        ds.load(file);
        return ds;
    }
}
 

@Override
public DataSet load(Source source) throws IOException {
    DataSet ds = new DataSet();
    try(InputStream is = source.getInputStream()){
        ds.load(is);
    }
    return ds;
}
 

protected DataSet load(PortableDataStream pds) {
    DataSet ds = new DataSet();
    try (InputStream is = pds.open()) {
        ds.load(is);
    } catch (IOException e) {
        throw new RuntimeException("Error loading DataSet at path " + pds.getPath() + " - DataSet may be corrupt or invalid." +
                " Spark DataSets can be validated using org.deeplearning4j.spark.util.data.SparkDataValidation", e);
    }
    cursor++;
    return ds;
}
 

@Override
public DataSet get(String key) {

    if (!cache.containsKey(key)) {
        return null;
    }

    byte[] data = cache.get(key);

    ByteArrayInputStream is = new ByteArrayInputStream(data);

    DataSet ds = new DataSet();

    ds.load(is);

    return ds;
}
 

@Override
public DataSet get(String key) {

    if (!cache.containsKey(key)) {
        return null;
    }

    byte[] data = cache.get(key);

    ByteArrayInputStream is = new ByteArrayInputStream(data);

    DataSet ds = new DataSet();

    ds.load(is);

    return ds;
}
 

@Override
protected DataSet load(File f) {
    DataSet ds = new DataSet();
    ds.load(f);
    return ds;
}
 

@Override
public <T> T call(File file) {
    DataSet dataSet = new DataSet();
    dataSet.load(file);
    return (T) dataSet;
}
 

@Override
public ValidationResult call(String path) throws Exception {
    if (fileSystem == null) {
        Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration();
        try {
            fileSystem = FileSystem.get(new URI(path), c);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    ValidationResult ret = new ValidationResult();
    ret.setCountTotal(1);

    boolean shouldDelete = false;
    boolean loadSuccessful = false;
    DataSet ds = new DataSet();
    Path p = new Path(path);

    if(fileSystem.isDirectory(p)){
        ret.setCountTotal(0);
        return ret;
    }

    if (!fileSystem.exists(p)) {
        ret.setCountMissingFile(1);
        return ret;
    }

    try (FSDataInputStream inputStream = fileSystem.open(p, BUFFER_SIZE)) {
        ds.load(inputStream);
        loadSuccessful = true;
    } catch (RuntimeException t) {
        shouldDelete = deleteInvalid;
        ret.setCountLoadingFailure(1);
    }

    boolean isValid = loadSuccessful;
    if (loadSuccessful) {
        //Validate
        if (ds.getFeatures() == null) {
            ret.setCountMissingFeatures(1);
            isValid = false;
        } else {
            if(featuresShape != null && !validateArrayShape(featuresShape, ds.getFeatures())){
                ret.setCountInvalidFeatures(1);
                isValid = false;
            }
        }

        if(ds.getLabels() == null){
            ret.setCountMissingLabels(1);
            isValid = false;
        } else {
            if(labelsShape != null && !validateArrayShape(labelsShape, ds.getLabels())){
                ret.setCountInvalidLabels(1);
                isValid = false;
            }
        }

        if(!isValid && deleteInvalid){
            shouldDelete = true;
        }
    }

    if (isValid) {
        ret.setCountTotalValid(1);
    } else {
        ret.setCountTotalInvalid(1);
    }

    if (shouldDelete) {
        fileSystem.delete(p, false);
        ret.setCountInvalidDeleted(1);
    }

    return ret;
}
 

@Test
    public void testBatchAndExportDataSetsFunction() throws Exception {
        String baseDir = System.getProperty("java.io.tmpdir");
        baseDir = FilenameUtils.concat(baseDir, "dl4j_spark_testBatchAndExport/");
        baseDir = baseDir.replaceAll("\\\\", "/");
        File f = new File(baseDir);
        if (f.exists())
            FileUtils.deleteDirectory(f);
        f.mkdir();
        f.deleteOnExit();
        int minibatchSize = 5;
        int nIn = 4;
        int nOut = 3;

        List<DataSet> dataSets = new ArrayList<>();
        dataSets.add(new DataSet(Nd4j.create(10, nIn), Nd4j.create(10, nOut))); //Larger than minibatch size -> tests splitting
        for (int i = 0; i < 98; i++) {
            if (i % 2 == 0) {
                dataSets.add(new DataSet(Nd4j.create(5, nIn), Nd4j.create(5, nOut)));
            } else {
                dataSets.add(new DataSet(Nd4j.create(1, nIn), Nd4j.create(1, nOut)));
                dataSets.add(new DataSet(Nd4j.create(1, nIn), Nd4j.create(1, nOut)));
                dataSets.add(new DataSet(Nd4j.create(3, nIn), Nd4j.create(3, nOut)));
            }
        }

        Collections.shuffle(dataSets, new Random(12345));

        JavaRDD<DataSet> rdd = sc.parallelize(dataSets);
        rdd = rdd.repartition(1); //For testing purposes (should get exactly 100 out, but maybe more with more partitions)


        JavaRDD<String> pathsRdd = rdd.mapPartitionsWithIndex(
                        new BatchAndExportDataSetsFunction(minibatchSize, "file:///" + baseDir), true);

        List<String> paths = pathsRdd.collect();
        assertEquals(100, paths.size());

        File[] files = f.listFiles();
        assertNotNull(files);

        int count = 0;
        for (File file : files) {
            if (!file.getPath().endsWith(".bin"))
                continue;
//            System.out.println(file);
            DataSet ds = new DataSet();
            ds.load(file);
            assertEquals(minibatchSize, ds.numExamples());

            count++;
        }

        assertEquals(100, count);

        FileUtils.deleteDirectory(f);
    }