org.elasticsearch.repositories.Repository Java Examples

/**
 * Returns a list of snapshots from repository sorted by snapshot creation date
 *
 * @param repositoryName repository name
 * @return list of snapshots
 */
public List<Snapshot> snapshots(String repositoryName, boolean ignoreUnavailable) {
    Set<Snapshot> snapshotSet = newHashSet();
    List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null);
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotSet.add(inProgressSnapshot(entry));
    }
    Repository repository = repositoriesService.repository(repositoryName);
    List<SnapshotId> snapshotIds = repository.snapshots();
    for (SnapshotId snapshotId : snapshotIds) {
        try {
            snapshotSet.add(repository.readSnapshot(snapshotId));
        } catch (Exception ex) {
            if (ignoreUnavailable) {
                logger.warn("failed to get snapshot [{}]", ex, snapshotId);
            } else {
                throw new SnapshotException(snapshotId, "Snapshot could not be read", ex);
            }
        }
    }

    ArrayList<Snapshot> snapshotList = new ArrayList<>(snapshotSet);
    CollectionUtil.timSort(snapshotList);
    return Collections.unmodifiableList(snapshotList);
}
 

@Override
public Map<String, Repository.Factory> getRepositories(Environment env,
                                                       NamedXContentRegistry namedXContentRegistry,
                                                       ThreadPool threadPool) {
    return Collections.singletonMap(
        URLRepository.TYPE,
        new Repository.Factory() {

            @Override
            public TypeSettings settings() {
                return new TypeSettings(URLRepository.mandatorySettings(), List.of());
            }

            @Override
            public Repository create(RepositoryMetaData metadata) throws Exception {
                return new URLRepository(metadata, env, namedXContentRegistry, threadPool);
            }
        }
    );
}
 

@Override
public Map<String, Repository.Factory> getRepositories(final Environment env, final NamedXContentRegistry registry, ThreadPool threadPool) {
    return Collections.singletonMap(
        S3Repository.TYPE,
        new Repository.Factory() {

            @Override
            public TypeSettings settings() {
                return new TypeSettings(List.of(), S3Repository.optionalSettings());
            }

            @Override
            public Repository create(RepositoryMetaData metadata) throws Exception {
                return new S3Repository(metadata, env.settings(), registry, service, threadPool);
            }
        }
    );
}
 

@Test
public void testUnavailableSnapshotsAreFilteredOut() {
    HashMap<String, SnapshotId> snapshots = new HashMap<>();
    SnapshotId s1 = new SnapshotId("s1", UUIDs.randomBase64UUID());
    SnapshotId s2 = new SnapshotId("s2", UUIDs.randomBase64UUID());
    snapshots.put(s1.getUUID(), s1);
    snapshots.put(s2.getUUID(), s2);
    RepositoryData repositoryData = new RepositoryData(
        1, snapshots, Collections.emptyMap(), Collections.emptyMap(), ShardGenerations.EMPTY);

    Repository r1 = mock(Repository.class);
    when(r1.getRepositoryData()).thenReturn(repositoryData);
    when(r1.getMetadata()).thenReturn(new RepositoryMetaData("repo1", "fs", Settings.EMPTY));
    when(r1.getSnapshotInfo(eq(s1))).thenThrow(new SnapshotException("repo1", "s1", "Everything is wrong"));
    when(r1.getSnapshotInfo(eq(s2))).thenReturn(new SnapshotInfo(s2, Collections.emptyList(), SnapshotState.SUCCESS));

    SysSnapshots sysSnapshots = new SysSnapshots(() -> Collections.singletonList(r1));
    List<SysSnapshot> currentSnapshots = ImmutableList.copyOf(sysSnapshots.currentSnapshots());
    assertThat(
        currentSnapshots.stream().map(SysSnapshot::name).collect(Collectors.toList()),
        containsInAnyOrder("s1", "s2")
    );
}
 

/**
 * Deletes a snapshot from the repository, looking up the {@link Snapshot} reference before deleting.
 * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
 *
 * @param repositoryName  repositoryName
 * @param snapshotName    snapshotName
 * @param listener        listener
 */
public void deleteSnapshot(final String repositoryName, final String snapshotName, final ActionListener<Void> listener,
                           final boolean immediatePriority) {
    // First, look for the snapshot in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    final RepositoryData repositoryData = repository.getRepositoryData();
    Optional<SnapshotId> matchedEntry = repositoryData.getSnapshotIds()
        .stream()
        .filter(s -> s.getName().equals(snapshotName))
        .findFirst();
    // if nothing found by the same name, then look in the cluster state for current in progress snapshots
    long repoGenId = repositoryData.getGenId();
    if (matchedEntry.isPresent() == false) {
        Optional<SnapshotsInProgress.Entry> matchedInProgress = currentSnapshots(repositoryName, Collections.emptyList()).stream()
            .filter(s -> s.snapshot().getSnapshotId().getName().equals(snapshotName)).findFirst();
        if (matchedInProgress.isPresent()) {
            matchedEntry = matchedInProgress.map(s -> s.snapshot().getSnapshotId());
            // Derive repository generation if a snapshot is in progress because it will increment the generation when it finishes
            repoGenId = matchedInProgress.get().getRepositoryStateId() + 1L;
        }
    }
    if (matchedEntry.isPresent() == false) {
        throw new SnapshotMissingException(repositoryName, snapshotName);
    }
    deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener, repoGenId, immediatePriority);
}
 

/**
 * Finalizes the shard in repository and then removes it from cluster state
 * <p>
 * This is non-blocking method that runs on a thread from SNAPSHOT thread pool
 *
 * @param entry   snapshot
 * @param failure failure reason or null if snapshot was successful
 */
private void endSnapshot(final SnapshotsInProgress.Entry entry, final String failure) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
        @Override
        public void run() {
            SnapshotId snapshotId = entry.snapshotId();
            try {
                final Repository repository = repositoriesService.repository(snapshotId.getRepository());
                logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshotId, entry.state(), failure);
                ArrayList<ShardSearchFailure> failures = new ArrayList<>();
                ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
                for (Map.Entry<ShardId, ShardSnapshotStatus> shardStatus : entry.shards().entrySet()) {
                    ShardId shardId = shardStatus.getKey();
                    ShardSnapshotStatus status = shardStatus.getValue();
                    if (status.state().failed()) {
                        failures.add(new ShardSearchFailure(status.reason(), new SearchShardTarget(status.nodeId(), shardId.getIndex(), shardId.id())));
                        shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId.getIndex(), shardId.id(), status.reason()));
                    }
                }
                Snapshot snapshot = repository.finalizeSnapshot(snapshotId, entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
                removeSnapshotFromClusterState(snapshotId, new SnapshotInfo(snapshot), null);
            } catch (Throwable t) {
                logger.warn("[{}] failed to finalize snapshot", t, snapshotId);
                removeSnapshotFromClusterState(snapshotId, null, t);
            }
        }
    });
}
 

@Override
public Map<String, Repository.Factory> getRepositories(Environment env,
                                                       NamedXContentRegistry namedXContentRegistry,
                                                       ThreadPool threadPool) {
    return Collections.singletonMap(
        AzureRepository.TYPE,
        new Repository.Factory() {

            @Override
            public TypeSettings settings() {
                return new TypeSettings(
                    AzureRepository.mandatorySettings(), AzureRepository.optionalSettings());
            }

            @Override
            public Repository create(RepositoryMetaData metadata) throws Exception {
                return new AzureRepository(
                    metadata,
                    env,
                    namedXContentRegistry,
                    azureStoreService,
                    threadPool
                );
            }
        }
    );
}
 

private RepositoryData getRepositoryData() throws Exception {
    RepositoriesService service = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    Repository repository = service.repository(REPOSITORY_NAME);
    ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName());
    final SetOnce<RepositoryData> repositoryData = new SetOnce<>();
    final CountDownLatch latch = new CountDownLatch(1);
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
        repositoryData.set(repository.getRepositoryData());
        latch.countDown();
    });
    latch.await();
    return repositoryData.get();
}
 

private static SysSnapshot createSysSnapshot(Repository repository, SnapshotId snapshotId) {
    SnapshotInfo snapshotInfo;
    try {
        snapshotInfo = repository.getSnapshotInfo(snapshotId);
    } catch (SnapshotException e) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Couldn't retrieve snapshotId={} error={}", snapshotId, e);
        }
        return new SysSnapshot(
            snapshotId.getName(),
            repository.getMetadata().name(),
            Collections.emptyList(),
            null,
            null,
            null,
            SnapshotState.FAILED.name(),
            List.of()
        );
    }
    Version version = snapshotInfo.version();
    return new SysSnapshot(
        snapshotId.getName(),
        repository.getMetadata().name(),
        snapshotInfo.indices(),
        snapshotInfo.startTime(),
        snapshotInfo.endTime(),
        version == null ? null : version.toString(),
        snapshotInfo.state().name(),
        Lists2.map(snapshotInfo.shardFailures(), SnapshotShardFailure::toString)
    );
}
 

/**
 * Deletes snapshot from repository
 *
 * @param snapshot   snapshot
 * @param listener   listener
 * @param repositoryStateId the unique id representing the state of the repository at the time the deletion began
 * @param version minimum ES version the repository should be readable by
 */
private void deleteSnapshotFromRepository(Snapshot snapshot, @Nullable ActionListener<Void> listener, long repositoryStateId,
                                          Version version) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> {
        Repository repository = repositoriesService.repository(snapshot.getRepository());
        repository.deleteSnapshot(snapshot.getSnapshotId(), repositoryStateId, version.onOrAfter(SHARD_GEN_IN_REPO_DATA_VERSION), ActionListener.wrap(v -> {
            LOGGER.info("snapshot [{}] deleted", snapshot);
            removeSnapshotDeletionFromClusterState(snapshot, null, l);
            }, ex -> removeSnapshotDeletionFromClusterState(snapshot, ex, l)
            ));
    }));
}
 

/**
 * Returns status of shards  currently finished snapshots
 * <p>
 * This method is executed on master node and it's complimentary to the
 * {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it
 * returns similar information but for already finished snapshots.
 * </p>
 *
 * @param repositoryName  repository name
 * @param snapshotInfo    snapshot info
 * @return map of shard id to snapshot status
 */
public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String repositoryName,
                                                             final RepositoryData repositoryData,
                                                             final SnapshotInfo snapshotInfo) throws IOException {
    final Repository repository = repositoriesService.repository(repositoryName);
    final Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
    for (String index : snapshotInfo.indices()) {
        IndexId indexId = repositoryData.resolveIndexId(index);
        IndexMetaData indexMetaData = repository.getSnapshotIndexMetaData(snapshotInfo.snapshotId(), indexId);
        if (indexMetaData != null) {
            int numberOfShards = indexMetaData.getNumberOfShards();
            for (int i = 0; i < numberOfShards; i++) {
                ShardId shardId = new ShardId(indexMetaData.getIndex(), i);
                SnapshotShardFailure shardFailure = findShardFailure(snapshotInfo.shardFailures(), shardId);
                if (shardFailure != null) {
                    shardStatus.put(shardId, IndexShardSnapshotStatus.newFailed(shardFailure.reason()));
                } else {
                    final IndexShardSnapshotStatus shardSnapshotStatus;
                    if (snapshotInfo.state() == SnapshotState.FAILED) {
                        // If the snapshot failed, but the shard's snapshot does
                        // not have an exception, it means that partial snapshots
                        // were disabled and in this case, the shard snapshot will
                        // *not* have any metadata, so attempting to read the shard
                        // snapshot status will throw an exception.  Instead, we create
                        // a status for the shard to indicate that the shard snapshot
                        // could not be taken due to partial being set to false.
                        shardSnapshotStatus = IndexShardSnapshotStatus.newFailed("skipped");
                    } else {
                        shardSnapshotStatus = repository.getShardSnapshotStatus(
                            snapshotInfo.snapshotId(),
                            indexId,
                            shardId);
                    }
                    shardStatus.put(shardId, shardSnapshotStatus);
                }
            }
        }
    }
    return unmodifiableMap(shardStatus);
}
 

/**
 * Returns a list of snapshots from repository sorted by snapshot creation date
 *
 * @param repositoryName repository name
 * @param snapshotIds       snapshots for which to fetch snapshot information
 * @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning,
 *                          if false, they will throw an error
 * @return list of snapshots
 */
public List<SnapshotInfo> snapshots(final String repositoryName,
                                    final List<SnapshotId> snapshotIds,
                                    final boolean ignoreUnavailable) {
    final Set<SnapshotInfo> snapshotSet = new HashSet<>();
    final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);
    // first, look at the snapshots in progress
    final List<SnapshotsInProgress.Entry> entries =
        currentSnapshots(repositoryName, snapshotIdsToIterate.stream().map(SnapshotId::getName).collect(Collectors.toList()));
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotSet.add(inProgressSnapshot(entry));
        snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId());
    }
    // then, look in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    for (SnapshotId snapshotId : snapshotIdsToIterate) {
        try {
            snapshotSet.add(repository.getSnapshotInfo(snapshotId));
        } catch (Exception ex) {
            if (ignoreUnavailable) {
                LOGGER.warn(() -> new ParameterizedMessage("failed to get snapshot [{}]", snapshotId), ex);
            } else {
                if (ex instanceof SnapshotException) {
                    throw ex;
                }
                throw new SnapshotException(repositoryName, snapshotId, "Snapshot could not be read", ex);
            }
        }
    }
    final ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet);
    CollectionUtil.timSort(snapshotList);
    return unmodifiableList(snapshotList);
}
 

/**
 * Restores shard from {@link SnapshotRecoverySource} associated with this shard in routing table
 */
private void restore(final IndexShard indexShard, final Repository repository, final SnapshotRecoverySource restoreSource) {
    final RecoveryState.Translog translogState = indexShard.recoveryState().getTranslog();
    if (restoreSource == null) {
        throw new IndexShardRestoreFailedException(shardId, "empty restore source");
    }
    if (logger.isTraceEnabled()) {
        logger.trace("[{}] restoring shard [{}]", restoreSource.snapshot(), shardId);
    }
    try {
        translogState.totalOperations(0);
        translogState.totalOperationsOnStart(0);
        indexShard.prepareForIndexRecovery();
        ShardId snapshotShardId = shardId;
        final String indexName = restoreSource.index();
        if (!shardId.getIndexName().equals(indexName)) {
            snapshotShardId = new ShardId(indexName, IndexMetaData.INDEX_UUID_NA_VALUE, shardId.id());
        }
        final IndexId indexId = repository.getRepositoryData().resolveIndexId(indexName);
        repository.restoreShard(indexShard.store(), restoreSource.snapshot().getSnapshotId(),
                                restoreSource.version(), indexId, snapshotShardId, indexShard.recoveryState());
        final Store store = indexShard.store();
        store.bootstrapNewHistory();
        final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo();
        final long maxSeqNo = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.MAX_SEQ_NO));
        final String translogUUID = Translog.createEmptyTranslog(
            indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm());
        store.associateIndexWithNewTranslog(translogUUID);
        assert indexShard.shardRouting.primary() : "only primary shards can recover from store";
        indexShard.openEngineAndRecoverFromTranslog();
        indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
        indexShard.finalizeRecovery();
        indexShard.postRecovery("restore done");
    } catch (Exception e) {
        throw new IndexShardRestoreFailedException(shardId, "restore failed", e);
    }
}
 

/**
 * Recovers an index from a given {@link Repository}. This method restores a
 * previously created index snapshot into an existing initializing shard.
 *
 * @param indexShard the index shard instance to recovery the snapshot from
 * @param repository the repository holding the physical files the shard should be recovered from
 * @return <code>true</code> if the shard has been recovered successfully, <code>false</code> if the recovery
 * has been ignored due to a concurrent modification of if the clusters state has changed due to async updates.
 */
boolean recoverFromRepository(final IndexShard indexShard, Repository repository) {
    if (canRecover(indexShard)) {
        RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType();
        assert recoveryType == RecoverySource.Type.SNAPSHOT : "expected snapshot recovery type: " + recoveryType;
        SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) indexShard.recoveryState().getRecoverySource();
        return executeRecovery(indexShard, () -> {
            logger.debug("restoring from {} ...", indexShard.recoveryState().getRecoverySource());
            restore(indexShard, repository, recoverySource);
        });
    }
    return false;

}
 

/**
 * Deletes snapshot from repository
 *
 * @param snapshotId snapshot id
 * @param listener   listener
 */
private void deleteSnapshotFromRepository(final SnapshotId snapshotId, final DeleteSnapshotListener listener) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
        @Override
        public void run() {
            try {
                Repository repository = repositoriesService.repository(snapshotId.getRepository());
                repository.deleteSnapshot(snapshotId);
                listener.onResponse();
            } catch (Throwable t) {
                listener.onFailure(t);
            }
        }
    });
}
 

@VisibleForTesting
SysSnapshots(Supplier<Collection<Repository>> getRepositories) {
    this.getRepositories = getRepositories;
}
 

public boolean restoreFromRepository(Repository repository) {
    assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard";
    assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT : "invalid recovery type: " + recoveryState.getRecoverySource();
    StoreRecovery storeRecovery = new StoreRecovery(shardId, logger);
    return storeRecovery.recoverFromRepository(this, repository);
}
 

@Override
public List<Setting<?>> getSettings() {
    return List.of(AzureRepository.Repository.ACCOUNT_SETTING, AzureRepository.Repository.KEY_SETTING);
}
 

/**
 * Returns repository types added by this plugin.
 *
 * @param env The environment for the local node, which may be used for the local settings and path.repo
 *
 * @param namedXContentRegistry the NamedXContentRegistry used for the {@link Repository.Factory}
 *
 * @param threadPool the thread pool used to execute the operations in the repositories
 *
 * The key of the returned {@link Map} is the type name of the repository and
 * the value is a factory to construct the {@link Repository} interface.
 */
default Map<String, Repository.Factory> getRepositories(Environment env,
                                                        NamedXContentRegistry namedXContentRegistry,
                                                        ThreadPool threadPool) {
    return Collections.emptyMap();
}