org.sonatype.nexus.repository.view.payloads.StreamPayload Java Examples

@Override
public void close() throws IOException {
  if (path != null) {
    mavenFacet.put(
        mavenPath,
        new StreamPayload(
            new InputStreamSupplier()
            {
              @Nonnull
              @Override
              public InputStream get() throws IOException {
                return new BufferedInputStream(Files.newInputStream(path));
              }
            },
            Files.size(path),
            contentType
        )
    );
    Files.delete(path);
    path = null;
  }
}
 

@Override
public Content handle(
    final Repository repository,
    final File content,
    final String path)
    throws IOException
{
  NpmHostedFacet npmFacet = repository.facet(NpmHostedFacet.class);
  StorageFacet storageFacet = repository.facet(StorageFacet.class);

  Path contentPath = content.toPath();
  Payload payload =
      new StreamPayload(() -> new FileInputStream(content), content.length(), Files.probeContentType(contentPath));
  TempBlob tempBlob = storageFacet.createTempBlob(payload, NpmFacetUtils.HASH_ALGORITHMS);
  final Map<String, Object> packageJson = npmPackageParser.parsePackageJson(tempBlob);
  ensureNpmPermitted(repository, packageJson);
  StorageTx tx = UnitOfWork.currentTx();
  Asset asset = npmFacet.putPackage(packageJson, tempBlob);
  return toContent(asset, tx.requireBlob(asset.requireBlobRef()));
}
 

@Override
public Content handle(
    final Repository repository,
    final File content,
    final String path)
    throws IOException
{
  MavenPath mavenPath = parser.parsePath(path);

  ensurePermitted(repository.getName(), Maven2Format.NAME, mavenPath.getPath(), toMap(mavenPath.getCoordinates()));

  if (mavenPath.getHashType() != null) {
    log.debug("skipping hash file {}", mavenPath);
    return null;
  }

  Path contentPath = content.toPath();
  Payload payload = new StreamPayload(() -> new FileInputStream(content), content.length(), Files.probeContentType(contentPath));
  MavenFacet mavenFacet = repository.facet(MavenFacet.class);
  Content asset = mavenFacet.put(mavenPath, payload);
  putChecksumFiles(mavenFacet, mavenPath, asset);
  return asset;
}
 

@Override
@Nullable
@Transactional
public Content getInfo(final String path,
                       final GolangAttributes golangAttributes)
{
  checkNotNull(path);
  checkNotNull(golangAttributes);
  String newPath = getZipAssetPathFromInfoPath(path);

  StreamPayload streamPayload = extractInfoFromZip(golangAttributes, newPath);
  if (streamPayload == null) {
    return null;
  }
  return new Content(streamPayload);
}
 

@TransactionalStoreBlob
public Asset ingestAsset(ControlFile control, TempBlob body, long size, String contentType) throws IOException, PGPException {
  AptFacet aptFacet = getRepository().facet(AptFacet.class);
  StorageTx tx = UnitOfWork.currentTx();
  Bucket bucket = tx.findBucket(getRepository());

  String name = control.getField("Package").map(f -> f.value).get();
  String version = control.getField("Version").map(f -> f.value).get();
  String architecture = control.getField("Architecture").map(f -> f.value).get();

  String assetPath = FacetHelper.buildAssetPath(name, version, architecture);

  Content content = aptFacet.put(assetPath, new StreamPayload(() -> body.get(), size, contentType));
  Asset asset = Content.findAsset(tx, bucket, content);
  String indexSection = buildIndexSection(control, asset.size(), asset.getChecksums(FacetHelper.hashAlgorithms), assetPath);
  asset.formatAttributes().set(P_ARCHITECTURE, architecture);
  asset.formatAttributes().set(P_PACKAGE_NAME, name);
  asset.formatAttributes().set(P_PACKAGE_VERSION, version);
  asset.formatAttributes().set(P_INDEX_SECTION, indexSection);
  asset.formatAttributes().set(P_ASSET_KIND, "DEB");
  tx.saveAsset(asset);

  List<AssetChange> changes = new ArrayList<>();
  changes.add(new AssetChange(AssetAction.ADDED, asset));

  for (Asset removed : selectOldPackagesToRemove(name, architecture)) {
    tx.deleteAsset(removed);
    changes.add(new AssetChange(AssetAction.REMOVED, removed));
  }

  rebuildIndexesInTransaction(tx, changes.stream().toArray(AssetChange[]::new));
  return asset;
}
 

@TransactionalStoreBlob
protected Asset ingestAsset(final ControlFile control, final TempBlob body, final long size, final String contentType) throws IOException {
  AptFacet aptFacet = getRepository().facet(AptFacet.class);
  StorageTx tx = UnitOfWork.currentTx();
  Bucket bucket = tx.findBucket(getRepository());

  PackageInfo info = new PackageInfo(control);
  String name = info.getPackageName();
  String version = info.getVersion();
  String architecture = info.getArchitecture();

  String assetPath = FacetHelper.buildAssetPath(name, version, architecture);

  Content content = aptFacet.put(
      assetPath,
      new StreamPayload(() -> body.get(), size, contentType),
      info);

  Asset asset = Content.findAsset(tx, bucket, content);
  String indexSection =
      buildIndexSection(control, asset.size(), asset.getChecksums(FacetHelper.hashAlgorithms), assetPath);
  asset.formatAttributes().set(P_ARCHITECTURE, architecture);
  asset.formatAttributes().set(P_PACKAGE_NAME, name);
  asset.formatAttributes().set(P_PACKAGE_VERSION, version);
  asset.formatAttributes().set(P_INDEX_SECTION, indexSection);
  asset.formatAttributes().set(P_ASSET_KIND, "DEB");
  tx.saveAsset(asset);

  rebuildIndexes(singletonList(new AssetChange(AssetAction.ADDED, asset)));
  return asset;
}
 

private static StreamPayload aStreamPayload(final Path path, final String contentType) throws IOException {
  return new StreamPayload(
      new InputStreamSupplier()
      {
        @Nonnull
        @Override
        public InputStream get() throws IOException {
          return new BufferedInputStream(Files.newInputStream(path));
        }
      },
      Files.size(path),
      contentType
  );
}
 

@Test
public void fileDoesNotExist() {
  StreamPayload payload = new StreamPayload(() -> getClass().getResourceAsStream(SONATYPE_ZIP),
      UNKNOWN_SIZE, ContentTypes.TEXT_PLAIN);

  assertThat(underTest.fileExists(payload, ANYPATH, "does_not_exist.mod"), is(false));
}
 

@Test
public void fileExists() {
  StreamPayload payload = new StreamPayload(() -> getClass().getResourceAsStream(SONATYPE_ZIP),
      UNKNOWN_SIZE, ContentTypes.TEXT_PLAIN);

  assertThat(underTest.fileExists(payload, ANYPATH, GO_MOD), is(true));
}
 

private StreamPayload extractInfoFromZip(final GolangAttributes goAttributes, final String newPath) {
  StorageTx tx = UnitOfWork.currentTx();

  Asset asset = golangDataAccess.findAsset(tx, tx.findBucket(getRepository()), newPath);
  if (asset == null) {
    return null;
  }

  return new StreamPayload(
      () -> doGetInfo(asset, goAttributes),
      UNKNOWN_SIZE,
      APPLICATION_JSON);
}
 

@Override
protected Content doPut(final Repository repository, final File content, final String path, final Path contentPath)
    throws IOException
{
  RawContentFacet facet = repository.facet(RawContentFacet.class);
  return new Content(facet.put(path, new StreamPayload(() -> new FileInputStream(content), Files.size(contentPath),
      Files.probeContentType(contentPath))));
}
 

@Override
protected Content doPut(final Repository repository, final File content, final String path, final Path contentPath)
    throws IOException
{
  RawContentFacet facet = repository.facet(RawContentFacet.class);
  return facet.put(path, new StreamPayload(() -> new FileInputStream(content), Files.size(contentPath),
      Files.probeContentType(contentPath)));
}
 

/**
 * Builds the index by querying (read only access) the underlying CMA structures.
 */
@Nonnull
@Override
protected Content buildIndex(final StorageTx tx, final Path path) throws IOException {
  Bucket bucket = tx.findBucket(getRepository());
  try (final Writer writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
    Set<NpmPackageId> packageIds = Sets.newHashSet(NpmFacetUtils.findAllPackageNames(tx, bucket));
    DateTime updated = new DateTime();
    JsonGenerator generator = NpmJsonUtils.mapper.getFactory().createGenerator(writer);
    generator.writeStartObject();
    generator.writeNumberField(NpmMetadataUtils.META_UPDATED, updated.getMillis());
    for (NpmPackageId packageId : packageIds) {
      final Asset packageRootAsset = NpmFacetUtils.findPackageRootAsset(tx, bucket, packageId);
      if (packageRootAsset != null) { // removed during iteration, skip
        NestedAttributesMap packageRoot = NpmFacetUtils.loadPackageRoot(tx, packageRootAsset);
        if (!packageRoot.isEmpty()) {
          NpmMetadataUtils.shrink(packageRoot);
          generator.writeObjectField(packageId.id(), packageRoot.backing());
        }
      }
    }
    generator.writeEndObject();
    generator.flush();
  }

  return new Content(new StreamPayload(
      new InputStreamSupplier()
      {
        @Nonnull
        @Override
        public InputStream get() throws IOException {
          return new BufferedInputStream(Files.newInputStream(path));
        }
      },
      Files.size(path),
      ContentTypes.APPLICATION_JSON)
  );
}
 

protected StreamPayload filePayload(final File file, final String contentType) {
  return new StreamPayload(
      new InputStreamSupplier()
      {
        @Nonnull
        @Override
        public InputStream get() throws IOException {
          return java.nio.file.Files.newInputStream(file.toPath());
        }
      },
      file.length(),
      contentType
  );
}
 

@TransactionalStoreMetadata
private void rebuildIndexesInTransaction(StorageTx tx, AssetChange... changes) throws IOException, PGPException {
  AptFacet aptFacet = getRepository().facet(AptFacet.class);
  AptSigningFacet signingFacet = getRepository().facet(AptSigningFacet.class);
  Bucket bucket = tx.findBucket(getRepository());

  StringBuilder sha256Builder = new StringBuilder();
  StringBuilder md5Builder = new StringBuilder();
  String releaseFile;
  try (CompressingTempFileStore store = buildPackageIndexes(tx, bucket, changes)) {
    for (Map.Entry<String, CompressingTempFileStore.FileMetadata> entry : store.getFiles().entrySet()) {
      Content plainContent = aptFacet.put(
          packageIndexName(entry.getKey(), ""),
          new StreamPayload(entry.getValue().plainSupplier(), entry.getValue().plainSize(), AptMimeTypes.TEXT));
      addSignatureItem(md5Builder, MD5, plainContent, packageRelativeIndexName(entry.getKey(), ""));
      addSignatureItem(sha256Builder, SHA256, plainContent, packageRelativeIndexName(entry.getKey(), ""));

      Content gzContent = aptFacet.put(
          packageIndexName(entry.getKey(), ".gz"),
          new StreamPayload(entry.getValue().gzSupplier(), entry.getValue().bzSize(), AptMimeTypes.GZIP));
      addSignatureItem(md5Builder, MD5, gzContent, packageRelativeIndexName(entry.getKey(), ".gz"));
      addSignatureItem(sha256Builder, SHA256, gzContent, packageRelativeIndexName(entry.getKey(), ".gz"));

      Content bzContent = aptFacet.put(
          packageIndexName(entry.getKey(), ".bz2"),
          new StreamPayload(entry.getValue().bzSupplier(), entry.getValue().bzSize(), AptMimeTypes.BZIP));
      addSignatureItem(md5Builder, MD5, bzContent, packageRelativeIndexName(entry.getKey(), ".bz2"));
      addSignatureItem(sha256Builder, SHA256, bzContent, packageRelativeIndexName(entry.getKey(), ".bz2"));
    }

    releaseFile = buildReleaseFile(aptFacet.getDistribution(), store.getFiles().keySet(), md5Builder.toString(), sha256Builder.toString());
  }

  aptFacet.put(releaseIndexName("Release"), new BytesPayload(releaseFile.getBytes(Charsets.UTF_8), AptMimeTypes.TEXT));
  byte[] inRelease = signingFacet.signInline(releaseFile);
  aptFacet.put(releaseIndexName("InRelease"), new BytesPayload(inRelease, AptMimeTypes.TEXT));
  byte[] releaseGpg = signingFacet.signExternal(releaseFile);
  aptFacet.put(releaseIndexName("Release.gpg"), new BytesPayload(releaseGpg, AptMimeTypes.SIGNATURE));
}
 

public Response get(final Context context) {
  log.debug("Request {}", context.getRequest().getPath());

  TokenMatcher.State state = context.getAttributes().require(TokenMatcher.State.class);
  ConanCoords coord = ConanHostedHelper.convertFromState(state);
  AssetKind assetKind = context.getAttributes().require(AssetKind.class);
  String assetPath = ConanHostedHelper.getHostedAssetPath(coord, assetKind);

  Content content = doGet(assetPath);
  if (content == null) {
    return HttpResponses.notFound();
  }

  return new Response.Builder()
      .status(success(OK))
      .payload(new StreamPayload(
          new InputStreamSupplier()
          {
            @Nonnull
            @Override
            public InputStream get() throws IOException {
              return content.openInputStream();
            }
          },
          content.getSize(),
          content.getContentType()))
      .build();
}
 

private Payload getModAsPayload(final Payload content, final String path) {
  return new StreamPayload(() -> doGetModAsStream(content, path),
      UNKNOWN_SIZE, TEXT_PLAIN);
}
 

/**
 * Filters the npm index document with given predicate/
 */
static Content filter(final Content fullIndex,
                      final Predicate<NestedAttributesMap> predicate) throws IOException
{
  checkNotNull(fullIndex);
  checkNotNull(predicate);
  final Path path = Files.createTempFile("npm-searchIndex-filter", "json");
  final Closer closer = Closer.create();
  try {
    final JsonParser jsonParser = mapper.getFactory().createParser(closer.register(fullIndex.openInputStream()));
    if (jsonParser.nextToken() == JsonToken.START_OBJECT &&
        NpmMetadataUtils.META_UPDATED.equals(jsonParser.nextFieldName())) {
      jsonParser.nextValue(); // skip value
    }
    final JsonGenerator generator = closer.register(
        mapper.getFactory().createGenerator(new BufferedOutputStream(Files.newOutputStream(path)))
    );
    generator.writeStartObject();
    generator.writeNumberField(NpmMetadataUtils.META_UPDATED, System.currentTimeMillis());
    NestedAttributesMap packageRoot;
    while ((packageRoot = getNext(jsonParser)) != null) {
      if (predicate.apply(packageRoot)) {
        generator.writeObjectField(packageRoot.getKey(), packageRoot.backing());
      }
    }
    generator.writeEndObject();
    generator.flush();
  }
  catch (Throwable t) {
    throw closer.rethrow(t);
  }
  finally {
    closer.close();
  }

  return new Content(new StreamPayload(
      new InputStreamSupplier()
      {
        @Nonnull
        @Override
        public InputStream get() throws IOException {
          return new BufferedInputStream(Files.newInputStream(path));
        }
      },
      Files.size(path),
      ContentTypes.APPLICATION_JSON)
  );
}
 

/**
 * Builds the index by merging member indexes with low memory footprint, as JSON indexes might get huge.
 */
@Nonnull
@Override
protected Content buildIndex(final StorageTx tx, final Path path) throws IOException {
  final List<Repository> members = facet(GroupFacet.class).leafMembers();
  final Closer closer = Closer.create();
  try {
    final ArrayList<JsonParser> parsers = new ArrayList<>(members.size());
    pauseTransactionAndProcessMemberRepositories(tx, members, closer, parsers);
    final JsonGenerator generator = closer.register(
        mapper.getFactory().createGenerator(new BufferedOutputStream(Files.newOutputStream(path)))
    );
    generator.writeStartObject();
    generator.writeNumberField(NpmMetadataUtils.META_UPDATED, System.currentTimeMillis());
    final PackageMerger packageMerger = new PackageMerger(parsers);
    while (!packageMerger.isDepleted()) {
      NestedAttributesMap packageRoot = packageMerger.next();
      generator.writeObjectField(packageRoot.getKey(), packageRoot.backing());
    }
    generator.writeEndObject();
    generator.flush();
  }
  catch (Throwable t) {
    throw closer.rethrow(t);
  }
  finally {
    closer.close();
  }

  return new Content(new StreamPayload(
      new InputStreamSupplier()
      {
        @Nonnull
        @Override
        public InputStream get() throws IOException {
          return new BufferedInputStream(Files.newInputStream(path));
        }
      },
      Files.size(path),
      ContentTypes.APPLICATION_JSON)
  );
}
 

@TransactionalStoreMetadata
public void rebuildIndexes(final List<AssetChange> changes) throws IOException {
  StorageTx tx = UnitOfWork.currentTx();
  AptFacet aptFacet = getRepository().facet(AptFacet.class);
  AptSigningFacet signingFacet = getRepository().facet(AptSigningFacet.class);
  Bucket bucket = tx.findBucket(getRepository());

  StringBuilder sha256Builder = new StringBuilder();
  StringBuilder md5Builder = new StringBuilder();
  String releaseFile;
  try (CompressingTempFileStore store = buildPackageIndexes(tx, bucket, changes)) {
    for (Map.Entry<String, CompressingTempFileStore.FileMetadata> entry : store.getFiles().entrySet()) {
      Content plainContent = aptFacet.put(
          packageIndexName(entry.getKey(), ""),
          new StreamPayload(entry.getValue().plainSupplier(), entry.getValue().plainSize(), AptMimeTypes.TEXT));
      addSignatureItem(md5Builder, MD5, plainContent, packageRelativeIndexName(entry.getKey(), ""));
      addSignatureItem(sha256Builder, SHA256, plainContent, packageRelativeIndexName(entry.getKey(), ""));

      Content gzContent = aptFacet.put(
          packageIndexName(entry.getKey(), ".gz"),
          new StreamPayload(entry.getValue().gzSupplier(), entry.getValue().bzSize(), AptMimeTypes.GZIP));
      addSignatureItem(md5Builder, MD5, gzContent, packageRelativeIndexName(entry.getKey(), ".gz"));
      addSignatureItem(sha256Builder, SHA256, gzContent, packageRelativeIndexName(entry.getKey(), ".gz"));

      Content bzContent = aptFacet.put(
          packageIndexName(entry.getKey(), ".bz2"),
          new StreamPayload(entry.getValue().bzSupplier(), entry.getValue().bzSize(), AptMimeTypes.BZIP));
      addSignatureItem(md5Builder, MD5, bzContent, packageRelativeIndexName(entry.getKey(), ".bz2"));
      addSignatureItem(sha256Builder, SHA256, bzContent, packageRelativeIndexName(entry.getKey(), ".bz2"));
    }

    releaseFile = buildReleaseFile(aptFacet.getDistribution(), store.getFiles().keySet(), md5Builder.toString(),
        sha256Builder.toString());
  }

  aptFacet.put(releaseIndexName(RELEASE), new BytesPayload(releaseFile.getBytes(Charsets.UTF_8), AptMimeTypes.TEXT));
  byte[] inRelease = signingFacet.signInline(releaseFile);
  aptFacet.put(releaseIndexName(INRELEASE), new BytesPayload(inRelease, AptMimeTypes.TEXT));
  byte[] releaseGpg = signingFacet.signExternal(releaseFile);
  aptFacet.put(releaseIndexName(RELEASE_GPG), new BytesPayload(releaseGpg, AptMimeTypes.SIGNATURE));
}