com.google.cloud.vision.v1.ImageContext Java Examples

@Test
public void testAddImageContext_analyzeImage() throws IOException {
	when(this.imageAnnotatorClient.batchAnnotateImages(any(BatchAnnotateImagesRequest.class)))
			.thenReturn(DEFAULT_API_RESPONSE);

	ImageContext imageContext = Mockito.mock(ImageContext.class);

	this.cloudVisionTemplate.analyzeImage(FAKE_IMAGE, imageContext, Type.FACE_DETECTION);

	BatchAnnotateImagesRequest expectedRequest =
			BatchAnnotateImagesRequest.newBuilder()
					.addRequests(
							AnnotateImageRequest.newBuilder()
									.addFeatures(Feature.newBuilder().setType(Type.FACE_DETECTION))
									.setImageContext(imageContext)
									.setImage(Image.newBuilder().setContent(
											ByteString.readFrom(FAKE_IMAGE.getInputStream())).build()))
					.build();

	verify(this.imageAnnotatorClient, times(1)).batchAnnotateImages(expectedRequest);
}
 

@Test
public void testAddImageContext_extractText() throws IOException {
	when(this.imageAnnotatorClient.batchAnnotateImages(any(BatchAnnotateImagesRequest.class)))
			.thenReturn(DEFAULT_API_RESPONSE);

	ImageContext imageContext = Mockito.mock(ImageContext.class);

	this.cloudVisionTemplate.extractTextFromImage(FAKE_IMAGE, imageContext);

	BatchAnnotateImagesRequest expectedRequest =
			BatchAnnotateImagesRequest.newBuilder()
					.addRequests(
							AnnotateImageRequest.newBuilder()
									.addFeatures(Feature.newBuilder().setType(Type.TEXT_DETECTION))
									.setImageContext(imageContext)
									.setImage(Image.newBuilder().setContent(ByteString.readFrom(FAKE_IMAGE.getInputStream())).build()))
					.build();

	verify(this.imageAnnotatorClient, times(1)).batchAnnotateImages(expectedRequest);
}
 

/**
 * Extract the text out of an image and return the result as a String.
 * @param imageResource the image one wishes to analyze
 * @param imageContext the image context to customize the text extraction request
 * @return the text extracted from the image aggregated to a String
 * @throws CloudVisionException if the image could not be read or if text extraction failed
 */
public String extractTextFromImage(Resource imageResource, ImageContext imageContext) {
	AnnotateImageResponse response = analyzeImage(imageResource, imageContext, Type.TEXT_DETECTION);

	String result = response.getFullTextAnnotation().getText();
	if (result.isEmpty() && response.getError().getCode() != Code.OK.getNumber()) {
		throw new CloudVisionException(response.getError().getMessage());
	}

	return result;
}
 

@ProcessElement
public void processElement(ProcessContext context) {
  if (sideInput != null) {
    Map<T, ImageContext> imageContextMap = context.sideInput(sideInput);
    context.output(mapToRequest(context.element(), imageContextMap.get(context.element())));
  } else {
    context.output(mapToRequest(context.element(), null));
  }
}
 

/**
 * @param contextSideInput Side input optionally containting a map of elements to {@link
 *     ImageContext} objects with metadata for the analysis.
 * @param featureList list of features to be extracted from the image.
 * @param batchSize desired size of request batches sent to Cloud Vision API. At least 1, at most
 *     16.
 * @param desiredRequestParallelism desiredRequestParallelism desired number of concurrent batched
 *     requests.
 */
public AnnotateImages(
    @Nullable PCollectionView<Map<T, ImageContext>> contextSideInput,
    List<Feature> featureList,
    long batchSize,
    int desiredRequestParallelism) {
  this.contextSideInput = contextSideInput;
  this.featureList = featureList;
  this.desiredRequestParallelism = desiredRequestParallelism;
  checkBatchSizeCorrectness(batchSize);
  this.batchSize = batchSize;
}
 

/**
 * Maps {@link KV} of {@link ByteString} (encoded image contents) and {@link ImageContext} to
 * {@link AnnotateImageRequest}.
 *
 * @param input Input element.
 * @param ctx optional image context.
 * @return valid request element.
 */
@Override
public AnnotateImageRequest mapToRequest(
    KV<ByteString, ImageContext> input, @Nullable ImageContext ctx) {
  Image image = Image.newBuilder().setContent(input.getKey()).build();
  AnnotateImageRequest.Builder builder =
      AnnotateImageRequest.newBuilder().setImage(image).addAllFeatures(featureList);
  if (input.getValue() != null) {
    builder.setImageContext(input.getValue());
  }
  return builder.build();
}
 

/**
 * Maps {@link KV} of {@link String} (GCS URI to the image) and {@link ImageContext} to a valid
 * {@link AnnotateImageRequest}.
 *
 * @param input Input element.
 * @param ctx optional image context, ignored here since the input holds context.
 * @return a valid request.
 */
@Override
public AnnotateImageRequest mapToRequest(
    KV<String, ImageContext> input, @Nullable ImageContext ctx) {
  ImageSource imageSource = ImageSource.newBuilder().setGcsImageUri(input.getKey()).build();
  Image image = Image.newBuilder().setSource(imageSource).build();
  AnnotateImageRequest.Builder builder =
      AnnotateImageRequest.newBuilder().setImage(image).addAllFeatures(featureList);
  if (input.getValue() != null) {
    builder.setImageContext(input.getValue());
  }
  return builder.build();
}
 

/**
 * Maps the {@link ByteString} with encoded image data and the optional {@link ImageContext}
 * into an {@link AnnotateImageRequest}.
 *
 * @param input Input element.
 * @param ctx optional image context.
 * @return a valid request.
 */
@Override
public AnnotateImageRequest mapToRequest(ByteString input, @Nullable ImageContext ctx) {
  AnnotateImageRequest.Builder builder = AnnotateImageRequest.newBuilder();
  if (ctx != null) {
    builder.setImageContext(ctx);
  }
  return builder
      .addAllFeatures(featureList)
      .setImage(Image.newBuilder().setContent(input).build())
      .build();
}
 

public AnnotateImagesFromBytes(
    PCollectionView<Map<ByteString, ImageContext>> contextSideInput,
    List<Feature> featureList,
    long batchSize,
    int desiredRequestParallelism) {
  super(contextSideInput, featureList, batchSize, desiredRequestParallelism);
}
 

/**
 * Maps the {@link String} with encoded image data and the optional {@link ImageContext} into an
 * {@link AnnotateImageRequest}.
 *
 * @param uri Input element.
 * @param ctx optional image context.
 * @return a valid request.
 */
@Override
public AnnotateImageRequest mapToRequest(String uri, @Nullable ImageContext ctx) {
  AnnotateImageRequest.Builder builder = AnnotateImageRequest.newBuilder();
  if (ctx != null) {
    builder.setImageContext(ctx);
  }
  ImageSource imgSource = ImageSource.newBuilder().setGcsImageUri(uri).build();
  return builder
      .addAllFeatures(featureList)
      .setImage(Image.newBuilder().setSource(imgSource).build())
      .build();
}
 

public AnnotateImagesFromGcsUri(
    PCollectionView<Map<String, ImageContext>> contextSideInput,
    List<Feature> featureList,
    long batchSize,
    int desiredRequestParallelism) {
  super(contextSideInput, featureList, batchSize, desiredRequestParallelism);
}
 

/**
 * Analyze an image and extract the features of the image specified by
 * {@code featureTypes}.
 * <p>A feature describes the kind of Cloud Vision analysis one wishes to perform on an
 * image, such as text detection, image labelling, facial detection, etc. A full list of
 * feature types can be found in {@link Feature.Type}.
 * @param imageResource the image one wishes to analyze. The Cloud Vision APIs support
 *     image formats described here: https://cloud.google.com/vision/docs/supported-files
 * @param imageContext the image context used to customize the Vision API request
 * @param featureTypes the types of image analysis to perform on the image
 * @return the results of image analyses
 * @throws CloudVisionException if the image could not be read or if a malformed response
 *     is received from the Cloud Vision APIs
 */
public AnnotateImageResponse analyzeImage(
		Resource imageResource, ImageContext imageContext, Feature.Type... featureTypes) {
	ByteString imgBytes;
	try {
		imgBytes = ByteString.readFrom(imageResource.getInputStream());
	}
	catch (IOException ex) {
		throw new CloudVisionException("Failed to read image bytes from provided resource.", ex);
	}

	Image image = Image.newBuilder().setContent(imgBytes).build();

	List<Feature> featureList = Arrays.stream(featureTypes)
			.map((featureType) -> Feature.newBuilder().setType(featureType).build())
			.collect(Collectors.toList());

	BatchAnnotateImagesRequest request = BatchAnnotateImagesRequest.newBuilder()
			.addRequests(
					AnnotateImageRequest.newBuilder()
							.addAllFeatures(featureList)
							.setImageContext(imageContext)
							.setImage(image))
			.build();

	BatchAnnotateImagesResponse batchResponse = this.imageAnnotatorClient.batchAnnotateImages(request);
	List<AnnotateImageResponse> annotateImageResponses = batchResponse.getResponsesList();

	if (!annotateImageResponses.isEmpty()) {
		return annotateImageResponses.get(0);
	}
	else {
		throw new CloudVisionException(
				"Failed to receive valid response Vision APIs; empty response received.");
	}
}
 

public MapInputToRequest(PCollectionView<Map<T, ImageContext>> sideInput) {
  this.sideInput = sideInput;
}
 

public static void detectWebEntitiesIncludeGeoResults(String filePath) throws IOException {

    // Initialize client that will be used to send requests. This client only needs to be created
    // once, and can be reused for multiple requests. After completing all of your requests, call
    // the "close" method on the client to safely clean up any remaining background resources.
    try (ImageAnnotatorClient client = ImageAnnotatorClient.create()) {
      // Read in the local image
      ByteString contents = ByteString.readFrom(new FileInputStream(filePath));

      // Build the image
      Image image = Image.newBuilder().setContent(contents).build();

      // Enable `IncludeGeoResults`
      WebDetectionParams webDetectionParams =
          WebDetectionParams.newBuilder().setIncludeGeoResults(true).build();

      // Set the parameters for the image
      ImageContext imageContext =
          ImageContext.newBuilder().setWebDetectionParams(webDetectionParams).build();

      // Create the request with the image, imageContext, and the specified feature: web detection
      AnnotateImageRequest request =
          AnnotateImageRequest.newBuilder()
              .addFeatures(Feature.newBuilder().setType(Type.WEB_DETECTION))
              .setImage(image)
              .setImageContext(imageContext)
              .build();

      // Perform the request
      BatchAnnotateImagesResponse response = client.batchAnnotateImages(Arrays.asList(request));

      // Display the results
      response.getResponsesList().stream()
          .forEach(
              r ->
                  r.getWebDetection().getWebEntitiesList().stream()
                      .forEach(
                          entity -> {
                            System.out.format("Description: %s%n", entity.getDescription());
                            System.out.format("Score: %f%n", entity.getScore());
                          }));
    }
  }
 

public static void detectWebEntitiesIncludeGeoResultsGcs(String gcsPath) throws IOException {

    // Initialize client that will be used to send requests. This client only needs to be created
    // once, and can be reused for multiple requests. After completing all of your requests, call
    // the "close" method on the client to safely clean up any remaining background resources.
    try (ImageAnnotatorClient client = ImageAnnotatorClient.create()) {
      // Set the image source to the given gs uri
      ImageSource imageSource = ImageSource.newBuilder().setGcsImageUri(gcsPath).build();
      // Build the image
      Image image = Image.newBuilder().setSource(imageSource).build();

      // Enable `IncludeGeoResults`
      WebDetectionParams webDetectionParams =
          WebDetectionParams.newBuilder().setIncludeGeoResults(true).build();

      // Set the parameters for the image
      ImageContext imageContext =
          ImageContext.newBuilder().setWebDetectionParams(webDetectionParams).build();

      // Create the request with the image, imageContext, and the specified feature: web detection
      AnnotateImageRequest request =
          AnnotateImageRequest.newBuilder()
              .addFeatures(Feature.newBuilder().setType(Feature.Type.WEB_DETECTION))
              .setImage(image)
              .setImageContext(imageContext)
              .build();

      // Perform the request
      BatchAnnotateImagesResponse response = client.batchAnnotateImages(Arrays.asList(request));

      // Display the results
      response.getResponsesList().stream()
          .forEach(
              r ->
                  r.getWebDetection().getWebEntitiesList().stream()
                      .forEach(
                          entity -> {
                            System.out.format("Description: %s%n", entity.getDescription());
                            System.out.format("Score: %f%n", entity.getScore());
                          }));
    }
  }
 

/**
 * Search similar products to image in local file.
 *
 * @param projectId - Id of the project.
 * @param computeRegion - Region name.
 * @param productSetId - Id of the product set.
 * @param productCategory - Category of the product.
 * @param filePath - Local file path of the image to be searched
 * @param filter - Condition to be applied on the labels. Example for filter: (color = red OR
 *     color = blue) AND style = kids It will search on all products with the following labels:
 *     color:red AND style:kids color:blue AND style:kids
 * @throws IOException - on I/O errors.
 */
public static void getSimilarProductsFile(
    String projectId,
    String computeRegion,
    String productSetId,
    String productCategory,
    String filePath,
    String filter)
    throws IOException {
  try (ImageAnnotatorClient queryImageClient = ImageAnnotatorClient.create()) {

    // Get the full path of the product set.
    String productSetPath =
        ProductSearchClient.formatProductSetName(projectId, computeRegion, productSetId);

    // Read the image as a stream of bytes.
    File imgPath = new File(filePath);
    byte[] content = Files.readAllBytes(imgPath.toPath());

    // Create annotate image request along with product search feature.
    Feature featuresElement = Feature.newBuilder().setType(Type.PRODUCT_SEARCH).build();
    // The input image can be a HTTPS link or Raw image bytes.
    // Example:
    // To use HTTP link replace with below code
    //  ImageSource source = ImageSource.newBuilder().setImageUri(imageUri).build();
    //  Image image = Image.newBuilder().setSource(source).build();
    Image image = Image.newBuilder().setContent(ByteString.copyFrom(content)).build();
    ImageContext imageContext =
        ImageContext.newBuilder()
            .setProductSearchParams(
                ProductSearchParams.newBuilder()
                    .setProductSet(productSetPath)
                    .addProductCategories(productCategory)
                    .setFilter(filter))
            .build();

    AnnotateImageRequest annotateImageRequest =
        AnnotateImageRequest.newBuilder()
            .addFeatures(featuresElement)
            .setImage(image)
            .setImageContext(imageContext)
            .build();
    List<AnnotateImageRequest> requests = Arrays.asList(annotateImageRequest);

    // Search products similar to the image.
    BatchAnnotateImagesResponse response = queryImageClient.batchAnnotateImages(requests);

    List<Result> similarProducts =
        response.getResponses(0).getProductSearchResults().getResultsList();
    System.out.println("Similar Products: ");
    for (Result product : similarProducts) {
      System.out.println(String.format("\nProduct name: %s", product.getProduct().getName()));
      System.out.println(
          String.format("Product display name: %s", product.getProduct().getDisplayName()));
      System.out.println(
          String.format("Product description: %s", product.getProduct().getDescription()));
      System.out.println(String.format("Score(Confidence): %s", product.getScore()));
      System.out.println(String.format("Image name: %s", product.getImage()));
    }
  }
}
 

/**
 * Search similar products to image in Google Cloud Storage.
 *
 * @param projectId - Id of the project.
 * @param computeRegion - Region name.
 * @param productSetId - Id of the product set.
 * @param productCategory - Category of the product.
 * @param gcsUri - GCS file path of the image to be searched
 * @param filter - Condition to be applied on the labels. Example for filter: (color = red OR
 *     color = blue) AND style = kids It will search on all products with the following labels:
 *     color:red AND style:kids color:blue AND style:kids
 * @throws Exception - on errors.
 */
public static void getSimilarProductsGcs(
    String projectId,
    String computeRegion,
    String productSetId,
    String productCategory,
    String gcsUri,
    String filter)
    throws Exception {
  try (ImageAnnotatorClient queryImageClient = ImageAnnotatorClient.create()) {

    // Get the full path of the product set.
    String productSetPath = ProductSetName.of(projectId, computeRegion, productSetId).toString();

    // Get the image from Google Cloud Storage
    ImageSource source = ImageSource.newBuilder().setGcsImageUri(gcsUri).build();

    // Create annotate image request along with product search feature.
    Feature featuresElement = Feature.newBuilder().setType(Type.PRODUCT_SEARCH).build();
    Image image = Image.newBuilder().setSource(source).build();
    ImageContext imageContext =
        ImageContext.newBuilder()
            .setProductSearchParams(
                ProductSearchParams.newBuilder()
                    .setProductSet(productSetPath)
                    .addProductCategories(productCategory)
                    .setFilter(filter))
            .build();

    AnnotateImageRequest annotateImageRequest =
        AnnotateImageRequest.newBuilder()
            .addFeatures(featuresElement)
            .setImage(image)
            .setImageContext(imageContext)
            .build();
    List<AnnotateImageRequest> requests = Arrays.asList(annotateImageRequest);

    // Search products similar to the image.
    BatchAnnotateImagesResponse response = queryImageClient.batchAnnotateImages(requests);

    List<Result> similarProducts =
        response.getResponses(0).getProductSearchResults().getResultsList();
    System.out.println("Similar Products: ");
    for (Result product : similarProducts) {
      System.out.println(String.format("\nProduct name: %s", product.getProduct().getName()));
      System.out.println(
          String.format("Product display name: %s", product.getProduct().getDisplayName()));
      System.out.println(
          String.format("Product description: %s", product.getProduct().getDescription()));
      System.out.println(String.format("Score(Confidence): %s", product.getScore()));
      System.out.println(String.format("Image name: %s", product.getImage()));
    }
  }
}
 

/**
 * Creates a {@link org.apache.beam.sdk.transforms.PTransform} that annotates images from their
 * contents encoded in {@link ByteString}s. Uses a default value 5 for desiredRequestParallelism.
 *
 * @param contextSideInput optional side input with contexts for select images. The {@link
 *     ImageContext} objects provide additional metadata for the annotation API. This way users
 *     can fine-tune analysis of selected images.
 * @param features annotation features that should be passed to the API
 * @param batchSize request batch size to be sent to API. Max 16, at least 1.
 * @return the PTransform.
 */
public static AnnotateImagesFromBytes annotateImagesFromBytes(
    PCollectionView<Map<ByteString, ImageContext>> contextSideInput,
    List<Feature> features,
    long batchSize) {
  return annotateImagesFromBytes(contextSideInput, features, batchSize, DEFAULT_PARALLELISM);
}
 

/**
 * Creates a {@link org.apache.beam.sdk.transforms.PTransform} that annotates images from their
 * contents encoded in {@link ByteString}s.
 *
 * @param contextSideInput optional side input with contexts for select images. The {@link
 *     ImageContext} objects provide additional metadata for the annotation API. This way users
 *     can fine-tune analysis of selected images.
 * @param features annotation features that should be passed to the API
 * @param batchSize request batch size to be sent to API. Max 16, at least 1.
 * @param desiredRequestParallelism desired number of concurrent batched requests.
 * @return the PTransform.
 */
public static AnnotateImagesFromBytes annotateImagesFromBytes(
    PCollectionView<Map<ByteString, ImageContext>> contextSideInput,
    List<Feature> features,
    long batchSize,
    int desiredRequestParallelism) {
  return new AnnotateImagesFromBytes(
      contextSideInput, features, batchSize, desiredRequestParallelism);
}
 

/**
 * Creates a {@link org.apache.beam.sdk.transforms.PTransform} that annotates images from their
 * GCS addresses. Uses a default value 5 for desiredRequestParallelism.
 *
 * @param contextSideInput optional side input with contexts for select images. The {@link
 *     ImageContext} objects provide additional metadata for the annotation API. This way users
 *     can fine-tune analysis of selected images.
 * @param features annotation features that should be passed to the API
 * @param batchSize request batch size to be sent to API. Max 16, at least 1.
 * @return the PTransform.
 */
public static AnnotateImagesFromGcsUri annotateImagesFromGcsUri(
    PCollectionView<Map<String, ImageContext>> contextSideInput,
    List<Feature> features,
    long batchSize) {
  return annotateImagesFromGcsUri(contextSideInput, features, batchSize, DEFAULT_PARALLELISM);
}
 

/**
 * Creates a {@link org.apache.beam.sdk.transforms.PTransform} that annotates images from their
 * GCS addresses.
 *
 * @param contextSideInput optional side input with contexts for select images. The {@link
 *     ImageContext} objects provide additional metadata for the annotation API. This way users
 *     can
 * @param features annotation features that should be passed to the API
 * @param batchSize request batch size to be sent to API. Max 16, at least 1.
 * @param desiredRequestParallelism desired number of concurrent batched requests.
 * @return the PTransform.
 */
public static AnnotateImagesFromGcsUri annotateImagesFromGcsUri(
    PCollectionView<Map<String, ImageContext>> contextSideInput,
    List<Feature> features,
    long batchSize,
    int desiredRequestParallelism) {
  return new AnnotateImagesFromGcsUri(
      contextSideInput, features, batchSize, desiredRequestParallelism);
}
 

/**
 * Analyze an image and extract the features of the image specified by
 * {@code featureTypes}.
 * <p>A feature describes the kind of Cloud Vision analysis one wishes to perform on an
 * image, such as text detection, image labelling, facial detection, etc. A full list of
 * feature types can be found in {@link Feature.Type}.
 * @param imageResource the image one wishes to analyze. The Cloud Vision APIs support
 *     image formats described here: https://cloud.google.com/vision/docs/supported-files
 * @param featureTypes the types of image analysis to perform on the image
 * @return the results of image analyses
 * @throws CloudVisionException if the image could not be read or if a malformed response
 *     is received from the Cloud Vision APIs
 */
public AnnotateImageResponse analyzeImage(Resource imageResource, Feature.Type... featureTypes) {
	return analyzeImage(imageResource, ImageContext.getDefaultInstance(), featureTypes);
}
 

/**
 * Input type to {@link AnnotateImageRequest} mapper. Needs to be implemented by child classes
 *
 * @param input Input element.
 * @param ctx optional image context.
 * @return A valid {@link AnnotateImageRequest} object.
 */
public abstract AnnotateImageRequest mapToRequest(T input, @Nullable ImageContext ctx);
 

/**
 * Extract the text out of an image and return the result as a String.
 * @param imageResource the image one wishes to analyze
 * @return the text extracted from the image aggregated to a String
 * @throws CloudVisionException if the image could not be read or if text extraction failed
 */
public String extractTextFromImage(Resource imageResource) {
	return extractTextFromImage(imageResource, ImageContext.getDefaultInstance());
}