org.datavec.api.io.labels.PathLabelGenerator Java Examples

public void load(long batchSize, long numExamples, long numLabels, PathLabelGenerator labelGenerator,
                double splitTrainTest, Random rng) {
    if (!imageFilesExist()) {
        if (!fullDir.exists() || fullDir.listFiles() == null || fullDir.listFiles().length == 0) {
            fullDir.mkdir();

            if (useSubset) {
                log.info("Downloading {} subset...", localDir);
                downloadAndUntar(lfwSubsetData, fullDir);
            } else {
                log.info("Downloading {}...", localDir);
                downloadAndUntar(lfwData, fullDir);
                downloadAndUntar(lfwLabel, fullDir);
            }
        }
    }
    FileSplit fileSplit = new FileSplit(fullDir, ALLOWED_FORMATS, rng);
    BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples,
                    numLabels, 0, batchSize, null);
    inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest));
}
 

public void load(int batchSize, int numExamples, int numLabels, PathLabelGenerator labelGenerator,
                double splitTrainTest, Random rng) {
    if (!imageFilesExist()) {
        if (!fullDir.exists() || fullDir.listFiles() == null || fullDir.listFiles().length == 0) {
            fullDir.mkdir();

            if (useSubset) {
                log.info("Downloading {} subset...", localDir);
                downloadAndUntar(lfwSubsetData, fullDir);
            } else {
                log.info("Downloading {}...", localDir);
                downloadAndUntar(lfwData, fullDir);
                downloadAndUntar(lfwLabel, fullDir);
            }
        }
    }
    FileSplit fileSplit = new FileSplit(fullDir, ALLOWED_FORMATS, rng);
    BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples,
                    numLabels, 0, batchSize, null);
    inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest));
}
 

protected BaseImageRecordReader(long height, long width, long channels, boolean nchw_channels_first, PathLabelGenerator labelGenerator,
                                PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
    this.height = height;
    this.width = width;
    this.channels = channels;
    this.labelGenerator = labelGenerator;
    this.labelMultiGenerator = labelMultiGenerator;
    this.imageTransform = imageTransform;
    this.appendLabel = (labelGenerator != null || labelMultiGenerator != null);
    this.nchw_channels_first = nchw_channels_first;
}
 

protected BaseImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator,
                                PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
    this.height = height;
    this.width = width;
    this.channels = channels;
    this.labelGenerator = labelGenerator;
    this.labelMultiGenerator = labelMultiGenerator;
    this.imageTransform = imageTransform;
    this.appendLabel = (labelGenerator != null || labelMultiGenerator != null);
}
 

public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator, int labelPosition) {
    this.selection = selection;
    this.mapper = mapper;
    this.shuffle = shuffle;
    this.rngSeed = rngSeed;
    if (shuffle)
        r = new Random(rngSeed);
    this.labelGenerator = labelGenerator;
    this.labelPosition = labelPosition;
}
 

public RecordReader getRecordReader(int batchSize, int numExamples, int[] imgDim, int numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        e.printStackTrace();
    }
    return recordReader;
}
 

public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator, int labelPosition) {
    this.selection = selection;
    this.mapper = mapper;
    this.shuffle = shuffle;
    this.rngSeed = rngSeed;
    if (shuffle)
        r = new Random(rngSeed);
    this.labelGenerator = labelGenerator;
    this.labelPosition = labelPosition;
}
 

public RecordReader getRecordReader(long batchSize, long numExamples, int[] imgDim, long numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        log.error("",e);
    }
    return recordReader;
}
 

public RecordReader getRecordReader(long batchSize, long numExamples, long[] imgDim, long numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        log.error("",e);
    }
    return recordReader;
}
 

@Test
public void testCsv() throws Exception {
    File extractedSourceDir = testDir.newFolder();
    new ClassPathResource("datavec-data-image/testimages").copyDirectory(extractedSourceDir);
    File baseDir = testDir.newFolder();


    List<File> c = new ArrayList<>(FileUtils.listFiles(extractedSourceDir, null, true));
    assertEquals(6, c.size());

    Collections.sort(c, new Comparator<File>() {
        @Override
        public int compare(File o1, File o2) {
            return o1.getPath().compareTo(o2.getPath());
        }
    });


    FileBatch fb = FileBatch.forFiles(c);
    File saveFile = new File(baseDir, "saved.zip");
    fb.writeAsZip(saveFile);
    fb = FileBatch.readFromZip(saveFile);

    PathLabelGenerator labelMaker = new ParentPathLabelGenerator();
    ImageRecordReader rr = new ImageRecordReader(32, 32, 1, labelMaker);
    rr.setLabels(Arrays.asList("class0", "class1"));
    FileBatchRecordReader fbrr = new FileBatchRecordReader(rr, fb);


    NativeImageLoader il = new NativeImageLoader(32, 32, 1);
    for( int test=0; test<3; test++) {
        for (int i = 0; i < 6; i++) {
            assertTrue(fbrr.hasNext());
            List<Writable> next = fbrr.next();
            assertEquals(2, next.size());

            INDArray exp;
            switch (i){
                case 0:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/0.jpg"));
                    break;
                case 1:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/1.png"));
                    break;
                case 2:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/2.jpg"));
                    break;
                case 3:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/A.jpg"));
                    break;
                case 4:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/B.png"));
                    break;
                case 5:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/C.jpg"));
                    break;
                default:
                    throw new RuntimeException();
            }
            Writable expLabel = (i < 3 ? new IntWritable(0) : new IntWritable(1));

            assertEquals(((NDArrayWritable)next.get(0)).get(), exp);
            assertEquals(expLabel, next.get(1));
        }
        assertFalse(fbrr.hasNext());
        assertTrue(fbrr.resetSupported());
        fbrr.reset();
    }
}
 

public BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator) {
    this(height, width, channels, labelGenerator, null);
}
 

public BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator,
                             ImageTransform imageTransform) {
    this(height, width, channels, labelGenerator, null, imageTransform);
}
 

protected BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator,
                                PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
    this(height, width, channels, true, labelGenerator, labelMultiGenerator, imageTransform);
}
 

public RecordReader getRecordReader(long batchSize, long numExamples, PathLabelGenerator labelGenerator,
                boolean train, double splitTrainTest, Random rng) {
    return getRecordReader(numExamples, batchSize, new long[] {height, width, channels},
                    useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator, train, splitTrainTest, rng);
}
 

public RecordReader getRecordReader(long batchSize, long numExamples, int[] imgDim, PathLabelGenerator labelGenerator,
                boolean train, double splitTrainTest, Random rng) {
    return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator,
                    train, splitTrainTest, rng);
}
 

public RecordReader getRecordReader(long batchSize, long numExamples, long[] imgDim, PathLabelGenerator labelGenerator,
                boolean train, double splitTrainTest, Random rng) {
    return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator,
                    train, splitTrainTest, rng);
}
 

@Test
public void testImageRecordReaderRegression() throws Exception {

    PathLabelGenerator regressionLabelGen = new TestRegressionLabelGen();

    ImageRecordReader rr = new ImageRecordReader(28, 28, 3, regressionLabelGen);

    File rootDir = testDir.newFolder();
    new ClassPathResource("datavec-data-image/testimages/").copyDirectory(rootDir);
    FileSplit fs = new FileSplit(rootDir);
    rr.initialize(fs);
    URI[] arr = fs.locations();

    assertTrue(rr.getLabels() == null || rr.getLabels().isEmpty());

    List<Writable> expLabels = new ArrayList<>();
    for(URI u : arr){
        String path = u.getPath();
        expLabels.add(testLabel(path.substring(path.length()-5, path.length())));
    }

    int count = 0;
    while(rr.hasNext()){
        List<Writable> l = rr.next();

        assertEquals(2, l.size());
        assertEquals(expLabels.get(count), l.get(1));

        count++;
    }
    assertEquals(6, count);

    //Test batch ops:
    rr.reset();

    List<List<Writable>> b1 = rr.next(3);
    List<List<Writable>> b2 = rr.next(3);
    assertFalse(rr.hasNext());

    NDArrayRecordBatch b1a = (NDArrayRecordBatch)b1;
    NDArrayRecordBatch b2a = (NDArrayRecordBatch)b2;
    assertEquals(2, b1a.getArrays().size());
    assertEquals(2, b2a.getArrays().size());

    NDArrayWritable l1 = new NDArrayWritable(Nd4j.create(new double[]{expLabels.get(0).toDouble(),
            expLabels.get(1).toDouble(), expLabels.get(2).toDouble()}, new long[]{3,1}, DataType.FLOAT));
    NDArrayWritable l2 = new NDArrayWritable(Nd4j.create(new double[]{expLabels.get(3).toDouble(),
            expLabels.get(4).toDouble(), expLabels.get(5).toDouble()}, new long[]{3,1}, DataType.FLOAT));

    INDArray act1 = b1a.getArrays().get(1);
    INDArray act2 = b2a.getArrays().get(1);
    assertEquals(l1.get(), act1);
    assertEquals(l2.get(), act2);
}
 

/** Loads images with given height, width, and channels, appending no labels. */
public ImageRecordReader(int height, int width, int channels) {
    super(height, width, channels, (PathLabelGenerator) null);
}
 

public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator) {
    this(selection, mapper, shuffle, rngSeed, labelGenerator, -1);
}
 

/** Calls {@code this(random, extensions, labelGenerator, 0, 0, 0, 0)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator) {
    this(random, extensions, labelGenerator, 0, 0, 0, 0);
}
 

/** Calls {@code this(random, null, labelGenerator, 0, 0, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, PathLabelGenerator labelGenerator, long maxPathsPerLabel) {
    this(random, null, labelGenerator, 0, 0, 0, maxPathsPerLabel);
}
 

/** Calls {@code this(random, extensions, labelGenerator, 0, 0, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator,
                long maxPathsPerLabel) {
    this(random, extensions, labelGenerator, 0, 0, 0, maxPathsPerLabel);
}
 

/** Calls {@code this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, PathLabelGenerator labelGenerator, long maxPaths, long maxLabels,
                long maxPathsPerLabel) {
    this(random, null, labelGenerator, maxPaths, maxLabels, 0, maxPathsPerLabel);
}
 

/** Calls {@code this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator, long maxLabels,
                long maxPathsPerLabel) {
    this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel);
}
 

/** Loads images with given  batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    this(batchSize, numExamples, imgDim, numLabels, useSubset, labelGenerator, train, splitTrainTest, null, rng);
}
 

/** Loads images with given height, width, and channels, appending labels returned by the generator.<br>
 * If {@code nchw_channels_first == true} output format is NCHW (channels first) - [numExamples, channels, height, width]<br>
 * If {@code nchw_channels_first == false} output format is NHWC (channels last) - [numExamples, height, width, channels]<br>
 */
public ImageRecordReader(long height, long width, long channels, boolean nchw_channels_first, PathLabelGenerator labelGenerator,
                         ImageTransform imageTransform) {
    super(height, width, channels, nchw_channels_first, labelGenerator, null, imageTransform);
}
 

/** Loads images with given height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator) {
    super(height, width, channels, labelGenerator);
}
 

/** Loads images with given height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator,
                ImageTransform imageTransform) {
    super(height, width, channels, labelGenerator, imageTransform);
}
 

/** Loads images with given  height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, PathLabelGenerator labelGenerator) {
    super(height, width, 1, labelGenerator);
}
 

public BaseImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator) {
    this(height, width, channels, labelGenerator, null);
}