org.sonatype.nexus.blobstore.api.BlobMetrics Java Examples

protected void writeTo(final Properties properties) {
  for (Entry<String, String> header : getHeaders().entrySet()) {
    properties.put(HEADER_PREFIX + header.getKey(), header.getValue());
  }
  BlobMetrics blobMetrics = getMetrics();
  properties.setProperty(SHA1_HASH_ATTRIBUTE, blobMetrics.getSha1Hash());
  properties.setProperty(CONTENT_SIZE_ATTRIBUTE, Long.toString(blobMetrics.getContentSize()));
  properties.setProperty(CREATION_TIME_ATTRIBUTE, Long.toString(blobMetrics.getCreationTime().getMillis()));

  if (deleted) {
    properties.put(DELETED_ATTRIBUTE, Boolean.toString(deleted));
    properties.put(DELETED_REASON_ATTRIBUTE, getDeletedReason());
    if (deletedDateTime != null) {
      properties.put(DELETED_DATETIME_ATTRIBUTE, Long.toString(deletedDateTime.getMillis()));
    }
  }
  else {
    properties.remove(DELETED_ATTRIBUTE);
    properties.remove(DELETED_REASON_ATTRIBUTE);
    properties.remove(DELETED_DATETIME_ATTRIBUTE);
  }
}
 

protected void readFrom(Properties properties) {
  headers = new HashMap<>();
  for (Entry<Object, Object> property : properties.entrySet()) {
    String key = (String) property.getKey();
    if (key.startsWith(HEADER_PREFIX)) {
      headers.put(key.substring(HEADER_PREFIX.length()), String.valueOf(property.getValue()));
    }
  }

  metrics = new BlobMetrics(
      new DateTime(Long.parseLong(properties.getProperty(CREATION_TIME_ATTRIBUTE))),
      properties.getProperty(SHA1_HASH_ATTRIBUTE),
      Long.parseLong(properties.getProperty(CONTENT_SIZE_ATTRIBUTE)));

  deleted = properties.containsKey(DELETED_ATTRIBUTE);
  deletedReason = properties.getProperty(DELETED_REASON_ATTRIBUTE);
  deletedDateTime = Optional.ofNullable(properties.getProperty(DELETED_DATETIME_ATTRIBUTE))
      .map(p -> new DateTime(Long.parseLong(p)))
      .orElse(null);
}
 

private void commonBlobMock(
    final int passes,
    final InputStream stream,
    final BlobMetrics blobMetrics,
    final String sha1)
{
  when(blobRef.getStore()).thenReturn("my-blobstore");

  BlobId blobId = new BlobId("blobId");
  when(blobRef.getBlobId()).thenReturn(blobId);
  when(blobStore.get(blobId)).thenReturn(blob);
  when(assetBlob.checksums()).thenReturn(Collections.singletonMap(HashAlgorithm.SHA1.name(), sha1));
  when(blob.getMetrics()).thenReturn(blobMetrics);
  when(blob.getInputStream()).thenReturn(stream);

}
 

private void readFrom(Properties properties) {
  headers = new HashMap<>();
  for (Entry<Object, Object> property : properties.entrySet()) {
    String key = (String) property.getKey();
    if (key.startsWith(HEADER_PREFIX)) {
      headers.put(key.substring(HEADER_PREFIX.length()), String.valueOf(property.getValue()));
    }
  }

  metrics = new BlobMetrics(
      new DateTime(Long.parseLong(properties.getProperty(CREATION_TIME_ATTRIBUTE))),
      properties.getProperty(SHA1_HASH_ATTRIBUTE),
      Long.parseLong(properties.getProperty(CONTENT_SIZE_ATTRIBUTE)));

  deleted = properties.containsKey(DELETED_ATTRIBUTE);
  deletedReason = properties.getProperty(DELETED_REASON_ATTRIBUTE);
}
 

/**
 * Verify that the Blob exists and is in agreement with the stored Asset metadata.;
 */
private void verifyBlob(final Blob blob, final Asset asset) throws MismatchedSHA1Exception, BlobUnavilableException, IOException {
  BlobMetrics metrics = blob.getMetrics();

  String assetChecksum =
      asset.blob().map(AssetBlob::checksums).map(checksums -> checksums.get(HashAlgorithm.SHA1.name())).orElse(null);
  if (!metrics.getSha1Hash().equals(assetChecksum)) {
    throw new MismatchedSHA1Exception();
  }

  try (InputStream blobstream = blob.getInputStream()) {
    if (metrics.getContentSize() > 0 && blobstream.available() == 0) {
      throw new BlobUnavilableException();
    }
  }
}
 

private AssetBlobData createAssetBlob(final BlobRef blobRef,
                                      final Blob blob,
                                      final Map<HashAlgorithm, HashCode> checksums)
{
  BlobMetrics metrics = blob.getMetrics();
  Map<String, String> headers = blob.getHeaders();

  AssetBlobData assetBlob = new AssetBlobData();
  assetBlob.setBlobRef(blobRef);
  assetBlob.setBlobSize(metrics.getContentSize());
  assetBlob.setContentType(headers.get(CONTENT_TYPE_HEADER));

  assetBlob.setChecksums(checksums.entrySet().stream().collect(
      toImmutableMap(
          e -> e.getKey().name(),
          e -> e.getValue().toString())));

  assetBlob.setBlobCreated(toOffsetDateTime(metrics.getCreationTime()));
  assetBlob.setCreatedBy(headers.get(CREATED_BY_HEADER));
  assetBlob.setCreatedByIp(headers.get(CREATED_BY_IP_HEADER));

  facet.stores().assetBlobStore.createAssetBlob(assetBlob);

  return assetBlob;
}
 

@Test
public void temporaryBlobMoveFallback() throws Exception {
  final byte[] content = new byte[TEST_DATA_LENGTH];
  new Random().nextBytes(content);

  doThrow(new AtomicMoveNotSupportedException("", "", "")).when(fileOperations).moveAtomic(any(), any());

  final Blob blob = underTest.create(new ByteArrayInputStream(content), TEST_HEADERS);
  verifyMoveOperations(blob);

  final byte[] output = extractContent(blob);
  assertThat("data must survive", content, is(equalTo(output)));

  final BlobMetrics metrics = blob.getMetrics();
  assertThat("size must be calculated correctly", metrics.getContentSize(), is(equalTo((long) TEST_DATA_LENGTH)));
}
 

private BlobAttributes getMockBlobAttribues(final String name, final String sha1, final boolean deleted) {
  BlobAttributes blobAttributes = mock(BlobAttributes.class);

  Properties properties = new Properties();
  if (name != null) {
    properties.setProperty(HEADER_PREFIX + BLOB_NAME_HEADER, name);
  }
  when(blobAttributes.getProperties()).thenReturn(properties);

  BlobMetrics metrics = new BlobMetrics(new DateTime(), sha1, 0);
  when(blobAttributes.getMetrics()).thenReturn(metrics);

  when(blobAttributes.isDeleted()).thenReturn(deleted);

  return blobAttributes;
}
 

@Test
public void testUpdateFrom() throws Exception {
  Path originalPath = temporaryFolder.newFile().toPath();

  Map<String, String> headers = ImmutableMap.of("hello", "world");
  BlobMetrics metrics = new BlobMetrics(DateTime.now(), "0123456789ABCDEF", 42);
  FileBlobAttributes original = new FileBlobAttributes(originalPath, headers, metrics);

  Path updatedPath = temporaryFolder.newFile().toPath();

  FileBlobAttributes updated = new FileBlobAttributes(updatedPath);
  updated.updateFrom(original);
  updated.store();

  updated = new FileBlobAttributes(updatedPath);
  updated.load();

  assertThat(updated.getHeaders(), is(original.getHeaders()));
  assertThat(updated.getMetrics().getCreationTime(), is(original.getMetrics().getCreationTime()));
  assertThat(updated.getMetrics().getSha1Hash(), is(original.getMetrics().getSha1Hash()));
  assertThat(updated.getMetrics().getContentSize(), is(original.getMetrics().getContentSize()));
  assertThat(updated.isDeleted(), is(original.isDeleted()));
  assertThat(updated.getDeletedReason(), is(original.getDeletedReason()));
}
 

private Blob createMockBlob(String id) {
  String createdBy = "createdBy";
  String createdByIp = "createdByIp";
  DateTime creationTime = new DateTime();
  Blob blob = mock(Blob.class);

  Map<String, String> headers = new HashMap<>();
  headers.put(CREATED_BY_HEADER, createdBy);
  headers.put(CREATED_BY_IP_HEADER, createdByIp);
  when(blob.getHeaders()).thenReturn(headers);

  BlobMetrics metrics = new BlobMetrics(creationTime, "hash", 1L);
  when(blob.getMetrics()).thenReturn(metrics);

  BlobId blobId = new BlobId(id);
  when(blob.getId()).thenReturn(blobId);
  when(blobStore.get(blobId)).thenReturn(blob);

  return blob;
}
 

@Override
@Guarded(by = STARTED)
public Blob copy(final BlobId blobId, final Map<String, String> headers) {
  GoogleCloudStorageBlob sourceBlob = (GoogleCloudStorageBlob) checkNotNull(get(blobId));

  return createInternal(headers, destination -> {
    sourceBlob.getBlob().copyTo(getConfiguredBucketName(), destination);
    BlobMetrics metrics = sourceBlob.getMetrics();
    return new StreamMetrics(metrics.getContentSize(), metrics.getSha1Hash());
  }, null);
}
 

protected BlobAttributesSupport(final T propertiesFile,
                                @Nullable final Map<String, String> headers,
                                @Nullable final BlobMetrics metrics) {
  this.propertiesFile = checkNotNull(propertiesFile);
  this.headers = headers;
  this.metrics = metrics;
}
 

@Test
public void doNotCheckAvailabilityOfInputStreamIfContentLengthIsZero() throws IOException {
  mockAssetBrowse();
  InputStream stream = mock(InputStream.class);
  commonBlobMock(1, stream, new BlobMetrics(DateTime.now(), "1234", 0));

  List<DeadBlobResult<Asset>> result = deadBlobFinder.find(repository);
  assertThat(result, empty());
  verify(stream, never()).available(); // never called as zero length blob would return 0 correctly
}
 

@Test
public void createAndDeleteBlobWithDirectPathSuccessful() throws IOException {
  final byte[] content = new byte[TEST_DATA_LENGTH];
  new Random().nextBytes(content);

  final Blob blob = underTest.create(new ByteArrayInputStream(content), ImmutableMap.of(
      CREATED_BY_HEADER, "test",
      BLOB_NAME_HEADER, "health-check/repositoryName/bundle.gz",
      DIRECT_PATH_BLOB_HEADER, "true"
  ));
  verifyMoveOperationsAtomic(blob);

  final byte[] output = extractContent(blob);
  assertThat("data must survive", content, is(equalTo(output)));

  final BlobMetrics metrics = blob.getMetrics();
  assertThat("size must be calculated correctly", metrics.getContentSize(), is(equalTo((long) TEST_DATA_LENGTH)));

  final BlobStoreMetrics storeMetrics = underTest.getMetrics();
  await().atMost(METRICS_FLUSH_TIMEOUT, SECONDS).until(() -> underTest.getMetrics().getBlobCount(), is(1L));

  assertThat(storeMetrics.getAvailableSpace(), is(greaterThan(0L)));

  final boolean deleted = underTest.delete(blob.getId(), "createAndDeleteBlobWithDirectPathSuccessful");
  assertThat(deleted, is(equalTo(true)));
  underTest.compact();
  await().atMost(METRICS_FLUSH_TIMEOUT, SECONDS).until(() -> underTest.getMetrics().getBlobCount(), is(0L));

  final Blob deletedBlob = underTest.get(blob.getId());
  assertThat(deletedBlob, is(nullValue()));
}
 

/**
 * Read all the content from a blob, and compare it with the metrics on file in the blob store.
 *
 * @throws RuntimeException if there is any deviation
 */
private void readContentAndValidateMetrics(final BlobId blobId,
                                           final InputStream inputStream,
                                           final BlobMetrics metadataMetrics)
    throws NoSuchAlgorithmException, IOException
{
  final MetricsInputStream measured = new MetricsInputStream(inputStream);
  ByteStreams.copy(measured, nullOutputStream());

  checkEqual("stream length", metadataMetrics.getContentSize(), measured.getSize(), blobId);
  checkEqual("SHA1 hash", metadataMetrics.getSha1Hash(), measured.getMessageDigest(), blobId);
}
 

@Nullable
private Long getContentSizeForDeletion(final BlobId blobId) {
  return Optional.ofNullable(getFileBlobAttributes(blobId))
        .map(BlobAttributes::getMetrics)
        .map(BlobMetrics::getContentSize)
        .orElse(null);
}
 

Blob createInternal(final Map<String, String> headers,
                    final BlobIngester ingester,
                    @Nullable final BlobId assignedBlobId)
{
  checkNotNull(headers);

  checkArgument(headers.containsKey(BLOB_NAME_HEADER), "Missing header: %s", BLOB_NAME_HEADER);
  checkArgument(headers.containsKey(CREATED_BY_HEADER), "Missing header: %s", CREATED_BY_HEADER);

  final BlobId blobId = getBlobId(headers, assignedBlobId);

  final String blobPath = contentPath(blobId);
  final String attributePath = attributePath(blobId);
  final GoogleCloudStorageBlob blob = liveBlobs.getUnchecked(blobId);
  Lock lock = blob.lock();
  try {
    log.debug("Writing blob {} to {}", blobId, blobPath);

    final StreamMetrics streamMetrics = ingester.ingestTo(blobPath);
    final BlobMetrics metrics = new BlobMetrics(new DateTime(), streamMetrics.getSha1(), streamMetrics.getSize());
    blob.refresh(headers, metrics);

    GoogleCloudBlobAttributes blobAttributes = new GoogleCloudBlobAttributes(bucket, attributePath, headers, metrics);

    blobAttributes.store();
    metricsStore.recordAddition(blobId, metrics.getContentSize());

    return blob;
  }
  catch (IOException e) {
    deleteNonExplosively(attributePath);
    deleteNonExplosively(blobPath);
    throw new BlobStoreException(e, blobId);
  }
  finally {
    lock.unlock();
  }
}
 

/**
 * Get the SHA1 from the {@link BlobAttributes}
 */
protected String getBlobSha1(final BlobAttributes blobAttributes) {
  BlobMetrics metrics = blobAttributes.getMetrics();
  checkArgument(metrics != null, "Blob attributes are missing metrics");
  String blobSha1 = metrics.getSha1Hash();
  checkArgument(blobSha1 != null, "Blob metrics are missing SHA1 hash code");
  return blobSha1;
}
 

@Override
@Guarded(by = STARTED)
@Timed
public Blob copy(final BlobId blobId, final Map<String, String> headers) {
  Blob sourceBlob = checkNotNull(get(blobId));
  String sourcePath = contentPath(sourceBlob.getId());
  return create(headers, destination -> {
      copier.copy(s3, getConfiguredBucket(), sourcePath, destination);
      BlobMetrics metrics = sourceBlob.getMetrics();
      return new StreamMetrics(metrics.getContentSize(), metrics.getSha1Hash());
  }, null);
}
 

@Override
@Guarded(by = STARTED)
public Blob copy(final BlobId blobId, final Map<String, String> headers) {
  Blob sourceBlob = checkNotNull(get(blobId));
  String sourcePath = contentPath(sourceBlob.getId());
  return create(headers, destination -> {
      s3.copyObject(getConfiguredBucket(), sourcePath, getConfiguredBucket(), destination);
      BlobMetrics metrics = sourceBlob.getMetrics();
      return new StreamMetrics(metrics.getContentSize(), metrics.getSha1Hash());
  });
}
 

private Blob create(final Map<String, String> headers, final BlobIngester ingester) {
  checkNotNull(headers);

  checkArgument(headers.containsKey(BLOB_NAME_HEADER), "Missing header: %s", BLOB_NAME_HEADER);
  checkArgument(headers.containsKey(CREATED_BY_HEADER), "Missing header: %s", CREATED_BY_HEADER);

  final BlobId blobId = blobIdLocationResolver.fromHeaders(headers);

  final String blobPath = contentPath(blobId);
  final String attributePath = attributePath(blobId);
  final S3Blob blob = liveBlobs.getUnchecked(blobId);

  Lock lock = blob.lock();
  try {
    log.debug("Writing blob {} to {}", blobId, blobPath);

    final StreamMetrics streamMetrics = ingester.ingestTo(blobPath);
    final BlobMetrics metrics = new BlobMetrics(new DateTime(), streamMetrics.getSha1(), streamMetrics.getSize());
    blob.refresh(headers, metrics);

    S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath, headers, metrics);

    blobAttributes.store();
    storeMetrics.recordAddition(blobAttributes.getMetrics().getContentSize());

    return blob;
  }
  catch (IOException e) {
    // Something went wrong, clean up the files we created
    deleteQuietly(attributePath);
    deleteQuietly(blobPath);
    throw new BlobStoreException(e, blobId);
  }
  finally {
    lock.unlock();
  }
}
 

private Properties writeTo(final Properties properties) {
  for (Entry<String, String> header : getHeaders().entrySet()) {
    properties.put(HEADER_PREFIX + header.getKey(), header.getValue());
  }
  BlobMetrics blobMetrics = getMetrics();
  properties.setProperty(SHA1_HASH_ATTRIBUTE, blobMetrics.getSha1Hash());
  properties.setProperty(CONTENT_SIZE_ATTRIBUTE, Long.toString(blobMetrics.getContentSize()));
  properties.setProperty(CREATION_TIME_ATTRIBUTE, Long.toString(blobMetrics.getCreationTime().getMillis()));

  if (deleted) {
    properties.put(DELETED_ATTRIBUTE, Boolean.toString(deleted));
    properties.put(DELETED_REASON_ATTRIBUTE, getDeletedReason());
  }
  return properties;
}
 

public GoogleCloudBlobAttributes(final Bucket bucket, final String key, final Map<String, String> headers,
                        final BlobMetrics metrics) {
  super(new GoogleCloudPropertiesFile(bucket, key), checkNotNull(headers), checkNotNull(metrics));
}
 

private Blob tryCreate(final Map<String, String> headers, final BlobIngester ingester, final BlobId reusedBlobId) { // NOSONAR
  final BlobId blobId = getBlobId(headers, reusedBlobId);
  final boolean isDirectPath = Boolean.parseBoolean(headers.getOrDefault(DIRECT_PATH_BLOB_HEADER, "false"));
  final Long existingSize = isDirectPath && exists(blobId) ? getContentSizeForDeletion(blobId) : null;

  final Path blobPath = contentPath(blobId);
  final Path attributePath = attributePath(blobId);

  final UUID uuidSuffix = UUID.randomUUID();
  final Path temporaryBlobPath = temporaryContentPath(blobId, uuidSuffix);
  final Path temporaryAttributePath = temporaryAttributePath(blobId, uuidSuffix);

  final FileBlob blob = liveBlobs.getUnchecked(blobId);

  Lock lock = blob.lock();
  try {
    final boolean wouldCollide = fileOperations.exists(blobPath);

    if ((reusedBlobId == null) && RETRY_ON_COLLISION && wouldCollide && !isDirectPath) {
      throw new BlobCollisionException(blobId);
    }
    try {
      log.debug("Writing blob {} to {}", blobId, blobPath);

      final StreamMetrics streamMetrics = ingester.ingestTo(temporaryBlobPath);
      final BlobMetrics metrics = new BlobMetrics(new DateTime(), streamMetrics.getSha1(), streamMetrics.getSize());
      blob.refresh(headers, metrics);

      // Write the blob attribute file
      FileBlobAttributes blobAttributes = new FileBlobAttributes(temporaryAttributePath, headers, metrics);
      blobAttributes.store();

      // Move the temporary files into their final location
      // existing size being not-null also implies isDirectPath is true
      if (existingSize != null) {
        overwrite(temporaryBlobPath, blobPath);
        overwrite(temporaryAttributePath, attributePath);
        metricsStore.recordDeletion(existingSize);
      }
      else {
        move(temporaryBlobPath, blobPath);
        move(temporaryAttributePath, attributePath);
      }

      metricsStore.recordAddition(blobAttributes.getMetrics().getContentSize());

      return blob;
    }
    catch (Exception e) {
      // Something went wrong, clean up the files we created
      fileOperations.deleteQuietly(temporaryAttributePath);
      fileOperations.deleteQuietly(temporaryBlobPath);
      fileOperations.deleteQuietly(attributePath);
      fileOperations.deleteQuietly(blobPath);
      throw new BlobStoreException(e, blobId);
    }
  }
  finally {
    lock.unlock();
  }
}
 

@Override
public BlobMetrics getMetrics() {
  return metrics;
}
 

public S3BlobAttributes(final AmazonS3 s3, final String bucket, final String key, final Map<String, String> headers,
                        final BlobMetrics metrics) {
  this(s3, bucket, key);
  this.headers = checkNotNull(headers);
  this.metrics = checkNotNull(metrics);
}
 

@Override
public BlobMetrics getMetrics() {
  return metrics;
}
 

public void refresh(final Map<String, String> headers, final BlobMetrics metrics) {
  this.headers = checkNotNull(headers);
  this.metrics = checkNotNull(metrics);
  stale = false;
}
 

@Override
public BlobMetrics getMetrics() {
  throw new UnsupportedOperationException("Not implemented");
}
 

@Override
public BlobMetrics getMetrics() {
  return metrics;
}