com.google.cloud.WriteChannel Java Examples

@Override
public void write(EncryptedKeyset keyset) throws IOException {
    BlobId blobId = BlobId.of(dekUri.getAuthority(), dekUri.getPath().substring(1));
    WriteChannel wc = CloudStorageUtils.getCloudStorageClient().writer(BlobInfo.newBuilder(blobId).build());
    OutputStream os = Channels.newOutputStream(wc);
    JsonKeysetWriter
        .withOutputStream(os)
        .write(keyset);
    os.close();
    wc.close();
}
 

@Override
public void uploadFile(final long size,
                       final InputStream localFileStream,
                       final RemoteObjectReference object) throws Exception {
    final BlobId blobId = ((GCPRemoteObjectReference) object).blobId;

    try (final WriteChannel outputChannel = storage.writer(BlobInfo.newBuilder(blobId).build(), Storage.BlobWriteOption.predefinedAcl(BUCKET_OWNER_FULL_CONTROL));
        final ReadableByteChannel inputChannel = Channels.newChannel(localFileStream)) {
        ByteStreams.copy(inputChannel, outputChannel);
    }
}
 

@Override
public void write(InputStream inputStream, String destination) throws IOException {
	String[] tokens = getBucketAndObjectFromPath(destination);
	Assert.state(tokens.length == 2, "Can only write to files, not buckets.");

	BlobInfo gcsBlobInfo = BlobInfo.newBuilder(BlobId.of(tokens[0], tokens[1])).build();

	try (InputStream is = inputStream) {
		try (WriteChannel channel = this.gcs.writer(gcsBlobInfo)) {
			channel.write(ByteBuffer.wrap(StreamUtils.copyToByteArray(is)));
		}
	}
}
 

@Test
public void testNewFiles() throws IOException {
	File testFile = this.temporaryFolder.newFile("benfica");

	BlobInfo expectedCreateBlobInfo =
			BlobInfo.newBuilder(BlobId.of("testGcsBucket", "benfica.writing")).build();
	WriteChannel writeChannel = mock(WriteChannel.class);
	willAnswer((invocationOnMock) -> writeChannel).given(GCS)
			.writer(eq(expectedCreateBlobInfo));
	willAnswer((invocationOnMock) -> 10).given(writeChannel).write(isA(ByteBuffer.class));

	CopyWriter copyWriter = mock(CopyWriter.class);
	ArgumentCaptor<Storage.CopyRequest> copyRequestCaptor = ArgumentCaptor.forClass(Storage.CopyRequest.class);
	willAnswer((invocationOnMock) -> copyWriter).given(GCS).copy(isA(Storage.CopyRequest.class));

	willAnswer((invocationOnMock) -> true).given(GCS)
			.delete(eq(BlobId.of("testGcsBucket", "benfica.writing")));

	this.channel.send(new GenericMessage<Object>(testFile));

	verify(GCS, times(1)).writer(eq(expectedCreateBlobInfo));
	verify(GCS, times(1)).copy(copyRequestCaptor.capture());
	verify(GCS, times(1)).delete(eq(BlobId.of("testGcsBucket", "benfica.writing")));

	Storage.CopyRequest expectedCopyRequest = copyRequestCaptor.getValue();
	assertThat(expectedCopyRequest.getSource()).isEqualTo(BlobId.of("testGcsBucket", "benfica.writing"));
	assertThat(expectedCopyRequest.getTarget().getBlobId()).isEqualTo(BlobId.of("testGcsBucket", "benfica"));
}
 

@Test
public void testWritable() throws Exception {
	WriteChannel writeChannel = mock(WriteChannel.class);
	when(this.mockStorage.writer(any(BlobInfo.class))).thenReturn(writeChannel);

	Assert.assertTrue(this.remoteResource instanceof WritableResource);
	WritableResource writableResource = (WritableResource) this.remoteResource;
	Assert.assertTrue(writableResource.isWritable());
	writableResource.getOutputStream();
}
 

@Test
public void testWritableOutputStream() throws Exception {
	String location = "gs://test-spring/test";
	Blob blob = mock(Blob.class);
	WriteChannel writeChannel = mock(WriteChannel.class);
	when(blob.writer()).thenReturn(writeChannel);
	when(blob.exists()).thenReturn(true);
	when(this.mockStorage.get(BlobId.of("test-spring", "test"))).thenReturn(blob);

	GoogleStorageResource resource = new GoogleStorageResource(this.mockStorage, location);
	OutputStream os = resource.getOutputStream();
	Assert.assertNotNull(os);
}
 

@Test
public void testWritableOutputStreamWithAutoCreateOnNullBlob() throws Exception {
	String location = "gs://test-spring/test";
	BlobId blobId = BlobId.of("test-spring", "test");
	when(this.mockStorage.get(blobId)).thenReturn(null);
	BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
	WriteChannel writeChannel = mock(WriteChannel.class);
	Blob blob = mock(Blob.class);
	when(blob.writer()).thenReturn(writeChannel);
	when(this.mockStorage.create(blobInfo)).thenReturn(blob);

	GoogleStorageResource resource = new GoogleStorageResource(this.mockStorage, location);
	GoogleStorageResource spyResource = spy(resource);
	OutputStream os = spyResource.getOutputStream();
	Assert.assertNotNull(os);
}
 

@Test
public void testWritableOutputStreamWithAutoCreateOnNonExistantBlob()
		throws Exception {
	String location = "gs://test-spring/test";
	BlobId blobId = BlobId.of("test-spring", "test");
	Blob nonExistantBlob = mock(Blob.class);
	when(nonExistantBlob.exists()).thenReturn(false);
	when(this.mockStorage.get(blobId)).thenReturn(nonExistantBlob);
	BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
	WriteChannel writeChannel = mock(WriteChannel.class);
	Blob blob = mock(Blob.class);
	when(blob.writer()).thenReturn(writeChannel);
	when(this.mockStorage.create(blobInfo)).thenReturn(blob);


	GoogleStorageResource resource = new GoogleStorageResource(this.mockStorage, location);
	GoogleStorageResource spyResource = spy(resource);
	OutputStream os = spyResource.getOutputStream();
	Assert.assertNotNull(os);
}
 

@PostMapping("/submitDocument")
public ModelAndView submitDocument(@RequestParam("documentUrl") String documentUrl) throws IOException {

	// Uploads the document to the GCS bucket
	Resource documentResource = resourceLoader.getResource(documentUrl);
	BlobId outputBlobId = BlobId.of(ocrBucket, documentResource.getFilename());
	BlobInfo blobInfo =
			BlobInfo.newBuilder(outputBlobId)
					.setContentType(getFileType(documentResource))
					.build();

	try (WriteChannel writer = storage.writer(blobInfo)) {
		ByteStreams.copy(documentResource.getInputStream(), Channels.newOutputStream(writer));
	}

	// Run OCR on the document
	GoogleStorageLocation documentLocation =
			GoogleStorageLocation.forFile(outputBlobId.getBucket(), outputBlobId.getName());

	GoogleStorageLocation outputLocation = GoogleStorageLocation.forFolder(
			outputBlobId.getBucket(), "ocr_results/" + documentLocation.getBlobName());

	ListenableFuture<DocumentOcrResultSet> result =
			documentOcrTemplate.runOcrForDocument(documentLocation, outputLocation);

	ocrStatusReporter.registerFuture(documentLocation.uriString(), result);

	return new ModelAndView("submit_done");
}
 

@Override
void run(BigQuery bigquery, Tuple<WriteChannelConfiguration, String> configuration)
    throws Exception {
  System.out.println("Running insert");
  try (FileChannel fileChannel = FileChannel.open(Paths.get(configuration.y()))) {
    WriteChannel writeChannel = bigquery.writer(configuration.x());
    long position = 0;
    long written = fileChannel.transferTo(position, CHUNK_SIZE, writeChannel);
    while (written > 0) {
      position += written;
      written = fileChannel.transferTo(position, CHUNK_SIZE, writeChannel);
    }
    writeChannel.close();
  }
}
 

/** Example of writing a blob's content through a writer. */
// [TARGET writer(BlobInfo, BlobWriteOption...)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name"]
public void writer(String bucketName, String blobName) throws IOException {
  // [START writer]
  BlobId blobId = BlobId.of(bucketName, blobName);
  byte[] content = "Hello, World!".getBytes(UTF_8);
  BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
  try (WriteChannel writer = storage.writer(blobInfo)) {
    try {
      writer.write(ByteBuffer.wrap(content, 0, content.length));
    } catch (Exception ex) {
      // handle exception
    }
  }
  // [END writer]
}
 

/** Example of writing the blob's content through a writer. */
// [TARGET writer(BlobWriteOption...)]
public void writer() throws IOException {
  // [START writer]
  byte[] content = "Hello, World!".getBytes(UTF_8);
  try (WriteChannel writer = blob.writer()) {
    try {
      writer.write(ByteBuffer.wrap(content, 0, content.length));
    } catch (Exception ex) {
      // handle exception
    }
  }
  // [END writer]
}
 

@Override
public void copyFromLocalFile(File srcFile, URI dstUri) throws Exception {
  LOGGER.info("Copying file {} to uri {}", srcFile.getAbsolutePath(), dstUri);
  Bucket bucket = getBucket(dstUri);
  Blob blob = bucket.create(sanitizePath(dstUri.getPath()), new byte[0]);
  WriteChannel writeChannel = blob.writer();
  writeChannel.setChunkSize(BUFFER_SIZE);
  ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
  SeekableByteChannel channel = Files.newByteChannel(srcFile.toPath());
  for (int bytesRead = channel.read(buffer); bytesRead != -1; bytesRead = channel.read(buffer)) {
    buffer.flip();
    writeChannel.write(buffer);
    buffer.clear();
  }
  writeChannel.close();
}
 

public static BlobId uploadToGcs(ByteBuffer content, String destinationUri, String contentType)
    throws Exception {
  URI uri = new URI(destinationUri);
  BlobId blobId = BlobId.of(uri.getHost(), uri.getPath().substring(1));
  BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(contentType).build();
  try (WriteChannel writer = storage.writer(blobInfo)) {
    writer.write(content);
  } catch (IOException e) {
    throw new UncheckedIOException(String.format("Failed to write to '%s'", destinationUri), e);
  }
  return blobId;
}
 

@Override
public Handle<?> upload(LogFilePath localPath) throws Exception {
    final String gsBucket = mConfig.getGsBucket();
    final String gsKey = localPath.withPrefix(mConfig.getGsPath()).getLogFilePath();
    final File localFile = new File(localPath.getLogFilePath());
    final boolean directUpload = mConfig.getGsDirectUpload();

    LOG.info("uploading file {} to gs://{}/{}", localFile, gsBucket, gsKey);

    final BlobInfo sourceBlob = BlobInfo
        .newBuilder(BlobId.of(gsBucket, gsKey))
        .setContentType(Files.probeContentType(localFile.toPath()))
        .build();

    rateLimiter.acquire();

    final Future<?> f = executor.submit(new Runnable() {
        @Override
        public void run() {
            try {
                if (directUpload) {
                    byte[] content = Files.readAllBytes(localFile.toPath());
                    Blob result = mClient.create(sourceBlob, content);
                    LOG.debug("Upload file {} to gs://{}/{}", localFile, gsBucket, gsKey);
                    LOG.trace("Upload file {}, Blob: {}", result);
                } else {
                    long startTime = System.nanoTime();
                    try (WriteChannel out = mClient.writer(sourceBlob);
                        FileChannel in = new FileInputStream(localFile).getChannel();
                        ) {
                        ByteBuffer buffer = ByteBuffer.allocateDirect(1024 * 1024 * 5); // 5 MiB buffer (remember this is pr. thread)

                        int bytesRead;
                        while ((bytesRead = in.read(buffer)) > 0) {
                            buffer.flip();
                            out.write(buffer);
                            buffer.clear();
                        }
                    }
                    long elapsedTime = System.nanoTime() - startTime;
                    LOG.debug("Upload file {} to gs://{}/{} in {} msec", localFile, gsBucket, gsKey, (elapsedTime / 1000000.0));
                }
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    });

    return new FutureHandle(f);
}