org.sonatype.nexus.blobstore.api.BlobStore Java Examples

@Override
public BlobStoreQuotaResult check(final BlobStore blobStore) {
  checkNotNull(blobStore);

  long availableSpace = blobStore.getMetrics().getAvailableSpace();
  boolean isUnlimited = blobStore.getMetrics().isUnlimited();
  long limit = getLimit(blobStore.getBlobStoreConfiguration());

  String name = blobStore.getBlobStoreConfiguration().getName();
  String msg = format("Blob store %s is limited to having %s available space, and has %s space remaining",
      name,
      formatStorage(limit),
      formatStorage(availableSpace));

  return new BlobStoreQuotaResult(!isUnlimited && availableSpace < limit, name, msg);
}
 

@Override
protected void doStop() throws Exception {
  if (stores.isEmpty()) {
    log.debug("No BlobStores defined");
    return;
  }

  log.debug("Stopping {} BlobStores", stores.size());
  for (Map.Entry<String, BlobStore> entry : stores.entrySet()) {
    String name = entry.getKey();
    BlobStore store = entry.getValue();
    log.debug("Stopping blob-store: {}", name);
    store.stop();

    // TODO - event publishing
  }

  stores.clear();
}
 

@PUT
@RequiresAuthentication
@Path("/{name}")
@RequiresPermissions("nexus:blobstores:update")
@Override
public GoogleCloudBlobstoreApiModel update(@PathParam("name") final String name,
                                           @Valid final GoogleCloudBlobstoreApiModel model)
    throws Exception
{
  BlobStore existing = blobStoreManager.get(name);
  if (existing == null) {
    return null;
  }
  BlobStoreConfiguration config = confirmType(existing.getBlobStoreConfiguration());
  merge(config, model);

  BlobStore blobStore = blobStoreManager.update(config);
  return new GoogleCloudBlobstoreApiModel(blobStore.getBlobStoreConfiguration());
}
 

private static InputStream packageRootAssetToInputStream(final Repository repository, final Asset packageRootAsset) {
  BlobStore blobStore = repository.facet(StorageFacet.class).blobStore();
  if (isNull(blobStore)) {
    throw new MissingAssetBlobException(packageRootAsset);
  }

  BlobRef blobRef = packageRootAsset.requireBlobRef();
  Blob blob = blobStore.get(blobRef.getBlobId());
  if (isNull(blob)) {
    throw new MissingAssetBlobException(packageRootAsset);
  }

  try {
    return blob.getInputStream();
  }
  catch (BlobStoreException ignore) { // NOSONAR
    // we want any issue with the blob store stream to be caught during the getting of the input stream as throw the
    // the same type of exception as a missing asset blob, so that we can pass the associated asset around.
    throw new MissingAssetBlobException(packageRootAsset);
  }
}
 

@Override
public void assertAssetMatchesBlob(final Repository repository, final String name) {
  try (StorageTx tx = getStorageTx(repository)) {
    tx.begin();
    Asset asset = tx.findAssetWithProperty(AssetEntityAdapter.P_NAME, name, tx.findBucket(repository));
    Blob blob = tx.requireBlob(asset.blobRef());

    assertThat(repository.getName(), equalTo(blob.getHeaders().get(Bucket.REPO_NAME_HEADER)));
    assertThat(asset.name(), equalTo(blob.getHeaders().get(BlobStore.BLOB_NAME_HEADER)));
    assertThat(asset.createdBy(), equalTo(blob.getHeaders().get(BlobStore.CREATED_BY_HEADER)));
    assertThat(asset.createdByIp(), equalTo(blob.getHeaders().get(BlobStore.CREATED_BY_IP_HEADER)));
    assertThat(asset.contentType(), equalTo(blob.getHeaders().get(BlobStore.CONTENT_TYPE_HEADER)));
    assertThat(asset.attributes().child("checksum").get("sha1"), equalTo(blob.getMetrics().getSha1Hash()));
    assertThat(asset.size(), equalTo(blob.getMetrics().getContentSize()));
  }
}
 

@Override
public void assertAssetMatchesBlob(final Repository repository, final String... names) {
  for (String name : names) {
    try (StorageTx tx = getStorageTx(repository)) {
      tx.begin();
      Asset asset = tx.findAssetWithProperty(AssetEntityAdapter.P_NAME, name, tx.findBucket(repository));
      Blob blob = tx.requireBlob(asset.blobRef());

      assertThat(repository.getName(), equalTo(blob.getHeaders().get(Bucket.REPO_NAME_HEADER)));
      assertThat(asset.name(), equalTo(blob.getHeaders().get(BlobStore.BLOB_NAME_HEADER)));
      assertThat(asset.createdBy(), equalTo(blob.getHeaders().get(BlobStore.CREATED_BY_HEADER)));
      assertThat(asset.createdByIp(), equalTo(blob.getHeaders().get(BlobStore.CREATED_BY_IP_HEADER)));
      assertThat(asset.contentType(), equalTo(blob.getHeaders().get(BlobStore.CONTENT_TYPE_HEADER)));
      assertThat(asset.attributes().child("checksum").get("sha1"), equalTo(blob.getMetrics().getSha1Hash()));
      assertThat(asset.size(), equalTo(blob.getMetrics().getContentSize()));
    }
  }
}
 

private void blobStoreIntegrityCheck(final boolean integrityCheck, final String blobStoreId) {
  if (!integrityCheck) {
    log.warn("Integrity check operation not selected");
    return;
  }

  BlobStore blobStore = blobStoreManager.get(blobStoreId);

  if (blobStore == null) {
    log.error("Unable to find blob store '{}' in the blob store manager", blobStoreId);
    return;
  }

  StreamSupport.stream(repositoryManager.browseForBlobStore(blobStoreId).spliterator(), false)
      .filter(r -> !(r.getType() instanceof GroupType))
      .forEach(repository -> integrityCheckStrategies
          .getOrDefault(repository.getFormat().getValue(), defaultIntegrityCheckStrategy)
          .check(repository, blobStore, this::isCanceled, this::integrityCheckFailedHandler)
      );
}
 

@Nonnull
private StorageTx openStorageTx(final ODatabaseDocumentTx db) {
  BlobStore blobStore = blobStoreManager.get(config.blobStoreName);
  return StateGuardAspect.around(
      new StorageTxImpl(
          createdBy(),
          createdByIp(),
          new BlobTx(nodeAccess, blobStore),
          db,
          getRepository().getName(),
          config.writePolicy == null ? WritePolicy.ALLOW : config.writePolicy,
          writePolicySelector,
          bucketEntityAdapter,
          componentEntityAdapter,
          assetEntityAdapter,
          config.strictContentTypeValidation,
          contentValidatorSelector.validator(getRepository()),
          mimeRulesSourceSelector.ruleSource(getRepository()),
          componentFactory
      )
  );
}
 

@Override
protected Result check() {
  Iterable<BlobStore> blobstoreItr = blobStoreManagerProvider.get().browse();
  final List<String> violationMessages = StreamSupport.stream(blobstoreItr.spliterator(), false)
      .map(blobStore -> quotaServiceProvider.get().checkQuota(blobStore))
      .filter(Objects::nonNull)
      .filter(BlobStoreQuotaResult::isViolation)
      .map(BlobStoreQuotaResult::getMessage)
      .collect(Collectors.toList());

  String message = format("%s/%s blob stores violating their quota<br>%s", violationMessages.size(),
      Iterators.size(blobStoreManagerProvider.get().browse().iterator()),
      String.join("<br>", violationMessages));

  return violationMessages.isEmpty() ? Result.healthy(message): Result.unhealthy(message);
}
 

@Override
@Guarded(by = ACTIVE)
public AssetBlob createBlob(final String blobName,
                            final Supplier<InputStream> streamSupplier,
                            final Iterable<HashAlgorithm> hashAlgorithms,
                            @Nullable final Map<String, String> headers,
                            @Nullable final String declaredContentType,
                            final boolean skipContentVerification) throws IOException
{
  checkNotNull(blobName);
  checkNotNull(streamSupplier);
  checkNotNull(hashAlgorithms);

  if (!writePolicy.checkCreateAllowed()) {
    throw new IllegalOperationException("Repository is read only: " + repositoryName);
  }

  Map<String, String> storageHeadersMap = buildStorageHeaders(blobName, streamSupplier, headers, declaredContentType,
      skipContentVerification);
  return blobTx.create(
      streamSupplier.get(),
      storageHeadersMap,
      hashAlgorithms,
      storageHeadersMap.get(BlobStore.CONTENT_TYPE_HEADER)
  );
}
 

@Subscribe
@AllowConcurrentEvents
public void on(final BlobStoreEvent event) {
  if (isRecording()) {
    BlobStore blobStore = event.getBlobStore();
    BlobStoreConfiguration configuration = blobStore.getBlobStoreConfiguration();

    AuditData data = new AuditData();
    data.setDomain(DOMAIN);
    data.setType(type(event.getClass()));
    data.setContext(configuration.getName());

    Map<String, Object> attributes = data.getAttributes();
    attributes.put("name", configuration.getName());
    attributes.put("type", configuration.getType());

    record(data);
  }
}
 

/**
 * Deletes the {@link AssetBlob} and associated {@link Blob}.
 *
 * @return {@code true} if the asset blob was deleted; otherwise {@code false}
 */
private boolean deleteAssetBlob(final AssetBlobStore<?> assetBlobStore, final BlobRef blobRef) {
  boolean assetBlobDeleted = false;

  BlobStore blobStore = blobStoreManager.get(blobRef.getStore());
  if (blobStore == null) {
    // postpone delete if the store is temporarily AWOL
    log.warn("Could not find blob store for {}", blobRef);
  }
  else {
    assetBlobDeleted = assetBlobStore.deleteAssetBlob(blobRef);
    if (assetBlobDeleted && !deleteBlobContent(blobStore, blobRef)) {
      log.warn("Could not delete blob content under {}", blobRef);
      // still report asset blob as deleted...
    }
  }

  return assetBlobDeleted;
}
 

@Override
@Guarded(by = STARTED)
public void forceDelete(final String name) throws Exception {
  checkNotNull(name);
  freezeService.checkWritable("Unable to delete a BlobStore while database is frozen.");

  BlobStore blobStore = blobStore(name);
  log.debug("Deleting BlobStore: {}", name);
  blobStore.shutdown();
  blobStore.remove();
  untrack(name);
  if (!EventHelper.isReplicating()) {
    store.delete(blobStore.getBlobStoreConfiguration());
  }
  eventManager.post(new BlobStoreDeletedEvent(blobStore));
}
 

private AssetBlob createAssetBlob(final Function<BlobStore, Blob> blobFunction,
                                  final Map<HashAlgorithm, HashCode> hashes,
                                  final boolean hashesVerified,
                                  final String contentType)
{
  AssetBlob assetBlob = new AssetBlob(
      nodeAccess,
      blobStore,
      blobFunction,
      contentType,
      hashes,
      hashesVerified);

  newlyCreatedBlobs.add(assetBlob);
  return assetBlob;
}
 

private void validateEligibleMembers(final String name, final List<String> memberNames) {
  for (String memberName : memberNames) {
    BlobStore member = blobStoreManager.get(memberName);
    if (!member.isGroupable()) {
      BlobStoreConfiguration memberConfig = member.getBlobStoreConfiguration();
      throw new ValidationErrorsException(
          format("Blob Store '%s' is of type '%s' and is not eligible to be a group member", memberName,
              memberConfig.getType()));
    }

    // target member may not be a member of a different group
    Predicate<String> sameGroup = name::equals;
    blobStoreManager.getParent(memberName).filter(sameGroup.negate()).ifPresent(groupName -> {
      throw new ValidationErrorsException(
          format("Blob Store '%s' is already a member of Blob Store Group '%s'", memberName, groupName));
    });

    // target member may not be set as repository storage
    int repoCount = blobStoreUtil.usageCount(memberName);
    if (repoCount > 0) {
      throw new ValidationErrorsException(format(
          "Blob Store '%s' is set as storage for %s repositories and is not eligible to be a group member",
          memberName, repoCount));
    }
  }
}
 

private void validateOnlyEmptyOrNotWritableExistingMembersRemoved(final String name, final List<String> memberNames) {
  BlobStore blobStore = blobStoreManager.get(name);
  if (blobStore != null) {
    BlobStoreConfiguration currentConfiguration = blobStore.getBlobStoreConfiguration();
    if (currentConfiguration != null && currentConfiguration.getType().equals(BlobStoreGroup.TYPE)) {
      for (String existingMemberName : memberNames(currentConfiguration)) {
        if (!memberNames.contains(existingMemberName)) {
          BlobStore existingMember = blobStoreManager.get(existingMemberName);
          if (existingMember.isWritable() || !existingMember.isEmpty()) {
            throw new ValidationErrorsException(
                format("Blob Store '%s' cannot be removed from Blob Store Group '%s', " +
                    "use 'Admin - Remove a member from a blob store group' task instead",
                    existingMemberName, name));
          }
        }
      }
    }
  }
}
 

@Nullable
@Override
@Guarded(by = STARTED)
public Blob get(final BlobId blobId, final boolean includeDeleted) {
  if (includeDeleted) {
    // check directly without using cache
    return members.get().stream()
        .map((BlobStore member) -> member.get(blobId, true))
        .filter(Objects::nonNull)
        .findAny()
        .orElse(null);
  }
  else {
    return locate(blobId)
      .map((BlobStore target) -> target.get(blobId, false))
      .orElse(null);
  }
}
 

@Test
public void createTempBlob_uploader() throws Exception {
  byte[] contents = "hello, world".getBytes(StandardCharsets.UTF_8);
  underTest.doConfigure(configuration);
  when(blobStoreManager.get(BLOB_STORE_NAME)).thenReturn(blobStore);
  when(blobStore.create(any(InputStream.class), Matchers.<Map<String, String>> any()))
      .thenAnswer(invocationOnMock -> {
        ByteStreams.toByteArray((InputStream) invocationOnMock.getArguments()[0]);
        return blob;
      });
  when(blobMetrics.getContentSize()).thenReturn((long) contents.length);
  when(clientInfoProvider.getCurrentThreadClientInfo()).thenReturn(clientInfo);
  when(clientInfo.getRemoteIP()).thenReturn("10.1.1.1");
  when(clientInfo.getUserid()).thenReturn("jpicard");
  when(payload.openInputStream()).thenAnswer(invocationOnMock -> new ByteArrayInputStream(contents));
  try (TempBlob tempBlob = underTest.createTempBlob(payload, singletonList(SHA1))) {
    verify(blobStore).create(any(InputStream.class), mapArgumentCaptor.capture());

    assertThat(mapArgumentCaptor.getValue(), hasEntry(BlobStore.CREATED_BY_IP_HEADER, "10.1.1.1"));
    assertThat(mapArgumentCaptor.getValue(), hasEntry(BlobStore.CREATED_BY_HEADER, "jpicard"));
  }
  verify(blobStore).deleteHard(blobId);
}
 

@VisibleForTesting
Optional<BlobStore> locate(final BlobId blobId) {
  String blobStoreName = locatedBlobs.get(blobId);
  if (blobStoreName != null) {
    log.trace("{} location was cached as {}", blobId, blobStoreName);
    return Optional.ofNullable(blobStoreManager.get(blobStoreName));
  }

  BlobStore blobStore = search(blobId);
  if (blobStore != null && blobStore.isWritable()) {
    String memberName = blobStore.getBlobStoreConfiguration().getName();
    log.trace("Caching {} in member {}", blobId, memberName);
    locatedBlobs.put(blobId, memberName);
  }

  return Optional.ofNullable(blobStore);
}
 

private BlobStore search(BlobId blobId) {
  log.trace("Searching for {} in {}", blobId, members);
  return members.get().stream()
    .sorted(Comparator.comparing(BlobStore::isWritable).reversed())
    .filter((BlobStore member) -> member.exists(blobId))
    .findAny()
    .orElse(null);
}
 

@Nullable
@Override
public BlobStoreQuotaResult checkQuota(final BlobStore blobStore) {
  checkNotNull(blobStore);
  BlobStoreConfiguration config = blobStore.getBlobStoreConfiguration();

  return getQuota(config).map(quota -> {
    log.debug("Checking blob store {} for quota {}", config.getName(), quota);
    return quota.check(blobStore);
  }).orElse(null);
}
 

@RequiresAuthentication
@RequiresPermissions("nexus:blobstores:read")
@GET
@Path("/{name}/quota-status")
public BlobStoreQuotaResultXO quotaStatus(@PathParam("name") final String name) {
  BlobStore blobStore = blobStoreManager.get(name);

  if (blobStore == null) {
    throw new WebApplicationException(format("No blob store found for id '%s' ", name), NOT_FOUND);
  }

  BlobStoreQuotaResult result = quotaService.checkQuota(blobStore);

  return result != null ? BlobStoreQuotaResultXO.asQuotaXO(result) : BlobStoreQuotaResultXO.asNoQuotaXO(name);
}
 

@Override
@Nullable
public BlobStore chooseBlobStore(final BlobStoreGroup blobStoreGroup, final Map<String, String> headers) {
  return blobStoreGroup
      .getMembers().stream()
      .filter(BlobStore::isWritable)
      .filter(BlobStore::isStorageAvailable)
      .findFirst()
      .orElse(null);
}
 

public BlobStore createGroup(String name, String... members) throws Exception {
  BlobStoreConfiguration config = blobStoreManagerProvider.get().newConfiguration();
  config.setName(name);
  config.setType(BlobStoreGroup.TYPE);
  config.attributes(BlobStoreGroup.CONFIG_KEY).set(BlobStoreGroup.MEMBERS_KEY, asList(members));
  config.attributes(BlobStoreGroup.CONFIG_KEY).set(BlobStoreGroup.FILL_POLICY_KEY, WriteToFirstMemberFillPolicy.TYPE);
  BlobStore blobStore = blobStoreManagerProvider.get().create(config);
  blobStoreGroupNames.add(name);
  return blobStore;
}
 

@Override
protected Object execute() throws Exception {
  BlobStore blobStore = blobStoreManager.get(getBlobStoreField());
  if (blobStore != null) {
    blobStore.compact(blobStoreUsageChecker);
  }
  else {
    log.warn("Unable to find blob store: {}", getBlobStoreField());
  }
  return null;
}
 

@VisibleForTesting
static void quotaCheckJob(final BlobStore blobStore, final BlobStoreQuotaService quotaService, final Logger logger) {
  try {
    BlobStoreQuotaResult result = quotaService.checkQuota(blobStore);
    if (result != null && result.isViolation()) {
      logger.warn(result.getMessage());
    }
  }
  catch (Exception e) {
    // Don't propagate, as this stops subsequent executions
    logger.error("Quota check exception for {}", blobStore.getBlobStoreConfiguration().getName(), e);
  }
}
 

private void detect(final BlobStore blobStore, final Consumer<String> handler) {
  Stream<BlobId> blobIds = blobStore.getBlobIdStream();

  blobIds.forEach(id -> {
    BlobAttributes attributes = blobStore.getBlobAttributes(id);
    if (attributes != null) {
      checkIfOrphaned(handler, id, attributes);
    }
    else{
      log.warn("Skipping cleanup for blob {} because blob properties not found", id);
    }
  });
}
 

private void delete(final BlobStore blobStore) {
  detect(blobStore, blobId -> {
    log.info("Deleting orphaned blob {} from blobstore {}", blobId, blobStore.getBlobStoreConfiguration().getName());

    blobStore.deleteHard(new BlobId(blobId));
  });
}
 

/**
 * Deletes the {@link Blob} from its {@link BlobStore}.
 *
 * @return {@code true} if the asset blob was deleted; otherwise {@code false}
 */
private boolean deleteBlobContent(BlobStore blobStore, final BlobRef blobRef) {
  if (HARD_DELETE) {
    return blobStore.deleteHard(blobRef.getBlobId());
  }
  else {
    return blobStore.delete(blobRef.getBlobId(), "Removing unused asset blob");
  }
}
 

@Override
@Guarded(by = STARTED)
public Blob copy(final BlobId blobId, final Map<String, String> headers) {
  BlobStore target = locate(blobId)
      .orElseThrow(() -> new BlobStoreException("Unable to find blob", blobId));
  Blob blob = target.copy(blobId, headers);
  locatedBlobs.put(blob.getId(), target.getBlobStoreConfiguration().getName());
  return blob;
}