Java Code Examples for org.datavec.image.loader.NativeImageLoader#asMatrix()

private INDArray imageToArray(InputStream imageIS, int height, int width, int channels) {
	NativeImageLoader loader = new NativeImageLoader(height, width, channels, true);
	INDArray imageArray = null;
	try {
		if (channels == 1) {
			imageArray = loader.asRowVector(imageIS);
		} else {
			imageArray = loader.asMatrix(imageIS);
		}
	} catch (Exception e) {
		throw new Error("Not able to convert image input stream to array.", e);
	}
	if (normalization != null) {
		normalization.transform(imageArray);
	}
	return imageArray;
}
 

public void trainModel(MultiLayerNetwork model, boolean invertColors, InputStream customImage, int customLabel) throws Exception {
    List<INDArray> extraFeatures = new LinkedList<>();
    List<Integer> extraLabels = new LinkedList<>();
    final INDArray[] customData = {null, null};
    if (customImage != null) {
        NativeImageLoader loader = new NativeImageLoader(width, height, channels);
        DataNormalization scaler = invertColors ? new ImagePreProcessingScaler(1, 0) : new ImagePreProcessingScaler(0, 1);
        customData[0] = loader.asMatrix(customImage);
        scaler.transform(customData[0]);
        customData[1] = Nd4j.create(1, 10);
        customData[1].putScalar(customLabel, 1.0);
        extraFeatures.add(customData[0]);
        extraLabels.add(customLabel);
    }
    trainModel(model, extraFeatures, extraLabels);
}
 

@Test
public void testImageNetLabels() throws IOException {
    // set up model
    ZooModel model = VGG19.builder().numClasses(0).build(); //num labels doesn't matter since we're getting pretrained imagenet
    ComputationGraph initializedModel = (ComputationGraph) model.initPretrained();

    // set up input and feedforward
    NativeImageLoader loader = new NativeImageLoader(224, 224, 3);
    ClassLoader classloader = Thread.currentThread().getContextClassLoader();
    INDArray image = loader.asMatrix(classloader.getResourceAsStream("deeplearning4j-zoo/goldenretriever.jpg"));
    DataNormalization scaler = new VGG16ImagePreProcessor();
    scaler.transform(image);
    INDArray[] output = initializedModel.output(false, image);

    // check output labels of result
    String decodedLabels = new ImageNetLabels().decodePredictions(output[0]);
    log.info(decodedLabels);
    assertTrue(decodedLabels.contains("golden_retriever"));

    // clean up for current model
    Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread();
    System.gc();
}
 

@Override
public void processValidWritable(Writable writable, List<Writable> record, int inputIndex, Object... extraArgs) {
    String inputName = imageLoadingStepConfig.getInputNames().get(inputIndex);

    NativeImageLoader nativeImageLoader = imageLoaders.get(inputName);

    ImageTransformProcess imageTransformProcess = null;
    if (imageLoadingStepConfig.getImageTransformProcesses() != null) {
        imageTransformProcess = imageLoadingStepConfig.getImageTransformProcesses().get(inputName);
    }

    INDArray input;

    try {
        if (writable instanceof ImageWritable) {
            input = nativeImageLoader.asMatrix(((ImageWritable) writable).getFrame());
        } else if (writable instanceof BytesWritable) {
            input = nativeImageLoader.asMatrix(((BytesWritable) writable).getContent());
        } else if (writable instanceof Text) {
            input = nativeImageLoader.asMatrix(writable.toString());
        } else if (writable instanceof NDArrayWritable) {
            input = ((NDArrayWritable) writable).get();
        } else {
            throw new IllegalArgumentException("Illegal type to load from " + writable.getClass());
        }

        INDArray output;

        if (imageLoadingStepConfig.isUpdateOrderingBeforeTransform()) {
            output = applyTransform(imageTransformProcess, nativeImageLoader, permuteImageOrder(input));
        } else {
            output = permuteImageOrder(applyTransform(imageTransformProcess, nativeImageLoader, input));
        }

        record.add(new NDArrayWritable(output));
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 

private INDArray imageFileToMatrix(File file) throws IOException {
    NativeImageLoader loader = new NativeImageLoader(224, 224, 3);
    INDArray image = loader.asMatrix(new FileInputStream(file));
    DataNormalization dataNormalization = new VGG16ImagePreProcessor();
    dataNormalization.transform(image);
    return image;
}
 

private INDArray prepareImage(BufferedImage convert, int width, int height) throws IOException {
    NativeImageLoader loader = new NativeImageLoader(height, width, 3);
    ImagePreProcessingScaler imagePreProcessingScaler = new ImagePreProcessingScaler(0, 1);

    INDArray indArray = loader.asMatrix(convert);
    if (indArray == null) {
        return null;
    }
    imagePreProcessingScaler.transform(indArray);
    return indArray;
}
 

/**
 * Describe <code>loadImage</code> method here.
 *
 * @param path a <code>File</code> value
 * @return an <code>INDArray</code> value
 * @exception IOException if an error occurs
 */
private INDArray loadImage(File path) throws IOException {
  int height = 60;
  int width = 200;
  int channels = 1;

  // height, width, channels
  NativeImageLoader loader = new NativeImageLoader(height, width, channels);
  INDArray image = loader.asMatrix(path);

  DataNormalization scaler = new ImagePreProcessingScaler(0, 1);
  scaler.transform(image);

  return image;
}
 

private INDArray imageToArray(InputStream imageIS, int height, int width, int channels) {
	NativeImageLoader loader = new NativeImageLoader(height, width, channels, true);
	INDArray imageArray = null;
	try {
		if (channels == 1) {
			imageArray = loader.asRowVector(imageIS);
		} else {
			imageArray = loader.asMatrix(imageIS);
		}
	} catch (Exception e) {
		throw new Error("Not able to convert image input stream to array.", e);
	}
	return imageArray;
}
 

public String outputForImageFile(File file, int h, int w, int channels) throws IOException {
	NativeImageLoader loader = new NativeImageLoader(h, w, channels);
	INDArray img1 = loader.asMatrix(file);
	if (model instanceof ComputationGraph) {
		((ComputationGraph) model).output(img1);
	} else if (model instanceof MultiLayerNetwork) {
		((MultiLayerNetwork) model).output(img1);
	}
	activationsCache.clear();
	return null;
}
 

public static void main(String[] args) throws Exception {
    String testPath = "data/test";
    File testDir = new File(testPath);
    File[] files = testDir.listFiles();

    Pair<MultiLayerNetwork, Normalizer> modelAndNormalizer = ModelSerializer
            .restoreMultiLayerNetworkAndNormalizer(new File("model/AlexNet.zip"), false);

    NativeImageLoader imageLoader = new NativeImageLoader(256, 256, 3);

    MultiLayerNetwork network = modelAndNormalizer.getFirst();
    DataNormalization normalizer = (DataNormalization) modelAndNormalizer.getSecond();

    Map<Integer, String> map = new HashMap<>();
    map.put(0, "CITY");
    map.put(1, "DESERT");
    map.put(2, "FARMLAND");
    map.put(3, "LAKE");
    map.put(4, "MOUNTAIN");
    map.put(5, "OCEAN");

    for (File file : files) {
        INDArray indArray = imageLoader.asMatrix(file);
        normalizer.transform(indArray);

        int[] values = network.predict(indArray);
        String label = map.get(values[0]);

        System.out.println(file.getName() + "," + label);
    }
}
 

public void trainModel(MultiLayerNetwork model, boolean invertColors, List<InputStream> customImage,
        List<Integer> customLabel) throws Exception {

    List<INDArray> extraFeatures = new LinkedList<>();
    List<Integer> extraLabels = new LinkedList<>();
    for (int i = 0; i < customImage.size(); i++) {
        NativeImageLoader loader = new NativeImageLoader(width, height, channels);
        DataNormalization scaler = invertColors ? new ImagePreProcessingScaler(1, 0) : new ImagePreProcessingScaler(0, 1);
        INDArray feature = loader.asMatrix(customImage.get(i));
        scaler.transform(feature);
        extraFeatures.add(feature);
        extraLabels.add(customLabel.get(i));
    }
    trainModel(model, extraFeatures, extraLabels);
}
 

public static void main(String[] args) throws IOException {
        /**
         * 首先我们需要创建一个数据读取器
         *
         * channel为1的时候，如果我们输入的一个彩色图，会自动的进行图像的灰度处理
         */
        NativeImageLoader imageLoader = new NativeImageLoader(height, width, channel);

        /**
         * 指定我们的图片位置
         */
        File imgFile = new ClassPathResource("/mnist/4.jpg").getFile();

        /**
         * 使用ImageLoader把图像转化为 INDArray
         */
        INDArray imgNdarray = imageLoader.asMatrix(imgFile);

//        DataNormalization scaler = new ImagePreProcessingScaler(0, 1);
//        scaler.transform(imgNdarray);

//        imgNdarray = imgNdarray.reshape(1, channel * height * width);

        /**
         * 打印查看数据
         */
//        System.out.println(imgNdarray);

        /**
         * 打印查看我们的数据的shape
         * 我们数据的相撞
         */
        System.out.println(Arrays.toString(imgNdarray.shape()));

        /**
         * 把灰度化之后的图片进行保存
         */
        ImageIO.write(imageFromINDArray(imgNdarray), "jpg", new File("model/test.jpg"));


        /**
         * 反序列化我们的模型
         * LeNet -> 卷积神经网络 -> 输入数据要求为4个维度 -> [batch, channel, height, width]
         * SingleLayer -> 全连接神经网络 -> 输入的数据要求的维度为2 -> [batch, features] -> [batch, channel * height * width]
         */
        MultiLayerNetwork network = ModelSerializer.restoreMultiLayerNetwork(modelPath, false);

        /**
         * 分类使用one-hot编码
         * 输出为概率
         * 概率最大的位置，为我们所分类的类型
         */
        INDArray output = network.output(imgNdarray);

        System.out.println(output);
        /**
         * 获取最大值的索引
         */
        System.out.println(Nd4j.getBlasWrapper().iamax(output));

        /**
         * output->只能输出模型的最后一层的经过激活函数的值 -> softmax
         * predict -> 其实就是一个 output 的封装 -> 只能用于分类
         */
        int[] results = network.predict(imgNdarray);
        System.out.println(Arrays.toString(results));

//        System.out.println(imageLoader.asMatrix(new File("model/test.jpg")));
    }
 

public static Thread train() {
    Thread t = new Thread() {
        @Override
        public void run() {
            while (true) {
                try {
                    List<TrainRequest> toProcess = new LinkedList<>();
                    synchronized (trainRequests) {
                        if (trainRequests.isEmpty()) {
                            System.out.println("Waiting for train requests...");
                            trainRequests.wait();
                            System.out.println("Got train requests...");

                        }
                        toProcess.addAll(trainRequests);
                        trainRequests.clear();
                    }
                    List<INDArray> features = new ArrayList<>(toProcess.size());
                    List<Integer> labels = new ArrayList<>(toProcess.size());
                    for (TrainRequest request : toProcess) {
                        NativeImageLoader loader = new NativeImageLoader(width, height, channels);
                        DataNormalization scaler = request.invert ? new ImagePreProcessingScaler(1, 0) : new ImagePreProcessingScaler(0, 1);
                        INDArray f = loader.asMatrix(new ByteArrayInputStream(request.b));
                        scaler.transform(f);
                        features.add(f);
                        labels.add(request.label);
                    }
                    MultiLayerNetwork result = trainModel(latestModel, features, labels);
                    if (callback != null) {
                        // this is synchronous!
                        System.out.println("invoking callback, Synchronous call!");
                        callback.accept(result);
                        System.out.println("invoked callback,");
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    };
    t.start();
    return t;
}
 

@Test
public void testCsv() throws Exception {
    File extractedSourceDir = testDir.newFolder();
    new ClassPathResource("datavec-data-image/testimages").copyDirectory(extractedSourceDir);
    File baseDir = testDir.newFolder();


    List<File> c = new ArrayList<>(FileUtils.listFiles(extractedSourceDir, null, true));
    assertEquals(6, c.size());

    Collections.sort(c, new Comparator<File>() {
        @Override
        public int compare(File o1, File o2) {
            return o1.getPath().compareTo(o2.getPath());
        }
    });


    FileBatch fb = FileBatch.forFiles(c);
    File saveFile = new File(baseDir, "saved.zip");
    fb.writeAsZip(saveFile);
    fb = FileBatch.readFromZip(saveFile);

    PathLabelGenerator labelMaker = new ParentPathLabelGenerator();
    ImageRecordReader rr = new ImageRecordReader(32, 32, 1, labelMaker);
    rr.setLabels(Arrays.asList("class0", "class1"));
    FileBatchRecordReader fbrr = new FileBatchRecordReader(rr, fb);


    NativeImageLoader il = new NativeImageLoader(32, 32, 1);
    for( int test=0; test<3; test++) {
        for (int i = 0; i < 6; i++) {
            assertTrue(fbrr.hasNext());
            List<Writable> next = fbrr.next();
            assertEquals(2, next.size());

            INDArray exp;
            switch (i){
                case 0:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/0.jpg"));
                    break;
                case 1:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/1.png"));
                    break;
                case 2:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/2.jpg"));
                    break;
                case 3:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/A.jpg"));
                    break;
                case 4:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/B.png"));
                    break;
                case 5:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/C.jpg"));
                    break;
                default:
                    throw new RuntimeException();
            }
            Writable expLabel = (i < 3 ? new IntWritable(0) : new IntWritable(1));

            assertEquals(((NDArrayWritable)next.get(0)).get(), exp);
            assertEquals(expLabel, next.get(1));
        }
        assertFalse(fbrr.hasNext());
        assertTrue(fbrr.resetSupported());
        fbrr.reset();
    }
}