org.elasticsearch.env.NodeEnvironment Java Examples

@Inject
public TransportNodesListGatewayStartedShards(Settings settings,
                                              ThreadPool threadPool,
                                              ClusterService clusterService,
                                              TransportService transportService,
                                              IndexNameExpressionResolver indexNameExpressionResolver,
                                              NodeEnvironment env,
                                              IndicesService indicesService,
                                              NamedXContentRegistry namedXContentRegistry) {
    super(
        ACTION_NAME,
        threadPool,
        clusterService,
        transportService,
        indexNameExpressionResolver,
        Request::new,
        NodeRequest::new,
        ThreadPool.Names.FETCH_SHARD_STARTED,
        NodeGatewayStartedShards.class
    );
    this.settings = settings;
    this.nodeEnv = env;
    this.indicesService = indicesService;
    this.namedXContentRegistry = namedXContentRegistry;
}
 

private boolean canDeleteShardContent(ShardId shardId, Settings indexSettings) {
    final IndexServiceInjectorPair indexServiceInjectorPair = this.indices.get(shardId.getIndex());
    if (IndexMetaData.isOnSharedFilesystem(indexSettings) == false) {
         if (nodeEnv.hasNodeFile()) {
            boolean isAllocated = indexServiceInjectorPair != null && indexServiceInjectorPair
                .getIndexService().hasShard(shardId.getId());
            if (isAllocated) {
                return false; // we are allocated - can't delete the shard
            }
            if (NodeEnvironment.hasCustomDataPath(indexSettings)) {
                // lets see if it's on a custom path (return false if the shared doesn't exist)
                // we don't need to delete anything that is not there
                return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId));
            } else {
                // lets see if it's path is available (return false if the shared doesn't exist)
                // we don't need to delete anything that is not there
                return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId));
            }
        }
    } else {
        logger.trace("{} skipping shard directory deletion due to shadow replicas", shardId);
    }
    return false;
}
 

@Override
public synchronized void assertAfterTest() throws IOException {
    super.assertAfterTest();
    assertRequestsFinished();
    for (NodeAndClient nodeAndClient : nodes.values()) {
        NodeEnvironment env = nodeAndClient.node().getNodeEnvironment();
        Set<ShardId> shardIds = env.lockedShards();
        for (ShardId id : shardIds) {
            try {
                env.shardLock(id, "InternalTestCluster assert after test", TimeUnit.SECONDS.toMillis(5)).close();
            } catch (ShardLockObtainFailedException ex) {
                fail("Shard " + id + " is still locked after 5 sec waiting");
            }
        }
    }
}
 

/**
 * This method tries to delete left-over shards where the index name has been reused but the UUID is different
 * to allow the new shard to be allocated.
 */
public static void deleteLeftoverShardDirectory(ESLogger logger, NodeEnvironment env, ShardLock lock, Settings indexSettings) throws IOException {
    final String indexUUID = indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE);
    final Path[] paths = env.availableShardPaths(lock.getShardId());
    for (Path path : paths) {
        ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
        if (load != null) {
            if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
                logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path);
                assert Files.isDirectory(path) : path + " is not a directory";
                NodeEnvironment.acquireFSLockForPaths(indexSettings, paths);
                IOUtils.rm(path);
            }
        }
    }
}
 

/** Maps each path.data path to a "guess" of how many bytes the shards allocated to that path might additionally use over their
 *  lifetime; we do this so a bunch of newly allocated shards won't just all go the path with the most free space at this moment. */
private static Map<Path,Long> getEstimatedReservedBytes(NodeEnvironment env, long avgShardSizeInBytes, Iterable<IndexShard> shards) throws IOException {
    long totFreeSpace = 0;
    for (NodeEnvironment.NodePath nodePath : env.nodePaths()) {
        totFreeSpace += nodePath.getUsableSpace();
    }

    // Very rough heurisic of how much disk space we expect the shard will use over its lifetime, the max of current average
    // shard size across the cluster and 5% of the total available free space on this node:
    long estShardSizeInBytes = Math.max(avgShardSizeInBytes, (long) (totFreeSpace/20.0));

    // Collate predicted (guessed!) disk usage on each path.data:
    Map<Path,Long> reservedBytes = new HashMap<>();
    for (IndexShard shard : shards) {
        Path dataPath = NodeEnvironment.shardStatePathToDataPath(shard.shardPath().getShardStatePath());

        // Remove indices/<index>/<shardID> subdirs from the statePath to get back to the path.data/<lockID>:
        Long curBytes = reservedBytes.get(dataPath);
        if (curBytes == null) {
            curBytes = 0L;
        }
        reservedBytes.put(dataPath, curBytes + estShardSizeInBytes);
    }       

    return reservedBytes;
}
 

protected void processNodePathsWithLock(Terminal terminal, OptionSet options, Environment env) throws IOException {
    terminal.println(Terminal.Verbosity.VERBOSE, "Obtaining lock for node");
    Integer nodeOrdinal = nodeOrdinalOption.value(options);
    if (nodeOrdinal == null) {
        nodeOrdinal = 0;
    }
    try (NodeEnvironment.NodeLock lock = new NodeEnvironment.NodeLock(nodeOrdinal, LOGGER, env, Files::exists)) {
        final Path[] dataPaths =
                Arrays.stream(lock.getNodePaths()).filter(Objects::nonNull).map(p -> p.path).toArray(Path[]::new);
        if (dataPaths.length == 0) {
            throw new ElasticsearchException(NO_NODE_FOLDER_FOUND_MSG);
        }
        processNodePaths(terminal, dataPaths, env);
    } catch (LockObtainFailedException ex) {
        throw new ElasticsearchException(
                FAILED_TO_OBTAIN_NODE_LOCK_MSG + " [" + ex.getMessage() + "]");
    }
}
 

/**
 * Runs an upgrade on all shards located under the given node environment if there is more than 1 data.path configured
 * otherwise this method will return immediately.
 */
public static void upgradeMultiDataPath(NodeEnvironment nodeEnv, ESLogger logger) throws IOException {
    if (nodeEnv.nodeDataPaths().length > 1) {
        final MultiDataPathUpgrader upgrader = new MultiDataPathUpgrader(nodeEnv);
        final Set<String> allIndices = nodeEnv.findAllIndices();

        for (String index : allIndices) {
            for (ShardId shardId : findAllShardIds(nodeEnv.indexPaths(new Index(index)))) {
                try (ShardLock lock = nodeEnv.shardLock(shardId, 0)) {
                    if (upgrader.needsUpgrading(shardId)) {
                        final ShardPath shardPath = upgrader.pickShardPath(shardId);
                        upgrader.upgrade(shardId, shardPath);
                        // we have to check if the index path exists since we might
                        // have only upgraded the shard state that is written under /indexname/shardid/_state
                        // in the case we upgraded a dedicated index directory index
                        if (Files.exists(shardPath.resolveIndex())) {
                            upgrader.checkIndex(shardPath);
                        }
                    } else {
                        logger.debug("{} no upgrade needed - already upgraded", shardId);
                    }
                }
            }
        }
    }
}
 

@Override
public Collection<Object> createComponents(Client client,
                                           ClusterService clusterService,
                                           ThreadPool threadPool,
                                           ResourceWatcherService resourceWatcherService,
                                           ScriptService scriptService,
                                           NamedXContentRegistry xContentRegistry,
                                           Environment environment,
                                           NodeEnvironment nodeEnvironment,
                                           NamedWriteableRegistry namedWriteableRegistry,
                                           IndexNameExpressionResolver indexNameExpressionResolver) {
    clusterService.addListener(event -> {
        for (Index i : event.indicesDeleted()) {
            if (IndexFeatureStore.isIndexStore(i.getName())) {
                caches.evict(i.getName());
            }
        }
    });
    return asList(caches, parserFactory);
}
 

static NodeEnvironment.NodePath getPathWithMostFreeSpace(NodeEnvironment env) throws IOException {
    final NodeEnvironment.NodePath[] paths = env.nodePaths();
    NodeEnvironment.NodePath bestPath = null;
    long maxUsableBytes = Long.MIN_VALUE;
    for (NodeEnvironment.NodePath nodePath : paths) {
        FileStore fileStore = nodePath.fileStore;
        long usableBytes = fileStore.getUsableSpace();
        assert usableBytes >= 0 : "usable bytes must be >= 0, got: " + usableBytes;

        if (bestPath == null || usableBytes > maxUsableBytes) {
            // This path has been determined to be "better" based on the usable bytes
            maxUsableBytes = usableBytes;
            bestPath = nodePath;
        }
    }
    return bestPath;
}
 

/**
 * This method tries to delete left-over shards where the index name has been reused but the UUID is different
 * to allow the new shard to be allocated.
 */
public static void deleteLeftoverShardDirectory(Logger logger, NodeEnvironment env, ShardLock lock, IndexSettings indexSettings) throws IOException {
    final String indexUUID = indexSettings.getUUID();
    final Path[] paths = env.availableShardPaths(lock.getShardId());
    for (Path path : paths) {
        // EMPTY is safe here because we never call namedObject
        ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
        if (load != null) {
            if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
                logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path);
                assert Files.isDirectory(path) : path + " is not a directory";
                NodeEnvironment.acquireFSLockForPaths(indexSettings, paths);
                IOUtils.rm(path);
            }
        }
    }
}
 

@Inject
public TransportNodesListShardStoreMetaData(Settings settings,
                                            ThreadPool threadPool,
                                            ClusterService clusterService,
                                            TransportService transportService,
                                            IndicesService indicesService,
                                            NodeEnvironment nodeEnv,
                                            IndexNameExpressionResolver indexNameExpressionResolver,
                                            NamedXContentRegistry namedXContentRegistry) {
    super(ACTION_NAME, threadPool, clusterService, transportService, indexNameExpressionResolver,
        Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STORE, NodeStoreFilesMetaData.class);
    this.settings = settings;
    this.indicesService = indicesService;
    this.nodeEnv = nodeEnv;
    this.namedXContentRegistry = namedXContentRegistry;
}
 

public AbstractTransportExportAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                     TransportService transportService, IndicesService indicesService,
                                     ScriptService scriptService, CacheRecycler cacheRecycler,
                                     IExportParser exportParser, Exporter exporter,
                                     NodeEnvironment nodeEnv) {
    super(settings, threadPool, clusterService, transportService);
    this.indicesService = indicesService;
    this.scriptService = scriptService;
    this.cacheRecycler = cacheRecycler;
    this.exportParser = exportParser;
    this.exporter = exporter;
    File[] paths = nodeEnv.nodeDataLocations();
    if (paths.length > 0) {
        nodePath = paths[0].getAbsolutePath();
    }
}
 

public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService,
                        MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader,
                        TransportService transportService, ClusterService clusterService,
                        IndicesService indicesService) throws IOException {
    this.settings = settings;
    this.nodeEnv = nodeEnv;
    this.metaStateService = metaStateService;
    this.transportService = transportService;
    this.clusterService = clusterService;
    this.indicesService = indicesService;

    upgradeMetaData(metaDataIndexUpgradeService, metaDataUpgrader);
    initializeClusterState(ClusterName.CLUSTER_NAME_SETTING.get(settings));
    incrementalWrite = false;
}
 

/**
 * Tries to open an index for the given location. This includes reading the
 * segment infos and possible corruption markers. If the index can not
 * be opened, an exception is thrown
 */
public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException, ShardLockObtainFailedException {
    try (ShardLock lock = shardLocker.lock(shardId, "open index", TimeUnit.SECONDS.toMillis(5));
         Directory dir = new SimpleFSDirectory(indexLocation)) {
        failIfCorrupted(dir, shardId);
        SegmentInfos segInfo = Lucene.readSegmentInfos(dir);
        logger.trace("{} loaded segment info [{}]", shardId, segInfo);
    }
}
 

@Override
public Collection<Object> createComponents(Client client,
                                           ClusterService clusterService,
                                           ThreadPool threadPool,
                                           NamedXContentRegistry xContentRegistry,
                                           Environment environment,
                                           NodeEnvironment nodeEnvironment,
                                           NamedWriteableRegistry namedWriteableRegistry) {
    // pipelineRegistry is returned here so that it's bound in guice and can be injected in other places
    return Collections.singletonList(pipelineRegistry);
}
 

@Inject
public DanglingIndicesState(NodeEnvironment nodeEnv, MetaStateService metaStateService,
                            LocalAllocateDangledIndices allocateDangledIndices, ClusterService clusterService) {
    this.nodeEnv = nodeEnv;
    this.metaStateService = metaStateService;
    this.allocateDangledIndices = allocateDangledIndices;
    clusterService.addListener(this);
}
 

public MockGatewayMetaState(Settings settings, NodeEnvironment nodeEnvironment,
                            NamedXContentRegistry xContentRegistry, DiscoveryNode localNode) throws IOException {
    super(settings, nodeEnvironment, new MetaStateService(nodeEnvironment, xContentRegistry),
            mock(MetaDataIndexUpgradeService.class), mock(MetaDataUpgrader.class),
            mock(TransportService.class), mock(ClusterService.class),
            mock(IndicesService.class));
    this.localNode = localNode;
}
 

private static NodeEnvironment.NodePath createNodePath(Path path) {
    try {
        return new NodeEnvironment.NodePath(path);
    } catch (IOException e) {
        throw new ElasticsearchException("Unable to investigate path [" + path + "]", e);
    }
}
 

public MonitorService(Settings settings,
                      NodeEnvironment nodeEnvironment,
                      ThreadPool threadPool,
                      ClusterInfoService clusterInfoService) {
    this.jvmGcMonitorService = new JvmGcMonitorService(settings, threadPool);
    this.osService = new OsService(settings);
    this.processService = new ProcessService(settings);
    this.jvmService = new JvmService(settings);
    this.fsService = new FsService(settings, nodeEnvironment, clusterInfoService);
}
 

/**
 * creates a new initializing shard. The shard will will be put in its proper path under the
 * current node id the shard is assigned to.
 * @param routing                shard routing to use
 * @param indexMetaData          indexMetaData for the shard, including any mapping
 * @param indexSearcherWrapper   an optional wrapper to be used during searchers
 * @param globalCheckpointSyncer callback for syncing global checkpoints
 * @param listeners              an optional set of listeners to add to the shard
 */
protected IndexShard newShard(ShardRouting routing, IndexMetaData indexMetaData,
                              @Nullable IndexSearcherWrapper indexSearcherWrapper,
                              @Nullable EngineFactory engineFactory,
                              Runnable globalCheckpointSyncer,
                              IndexingOperationListener... listeners)
    throws IOException {
    // add node id as name to settings for proper logging
    final ShardId shardId = routing.shardId();
    final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir());
    ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
    return newShard(routing, shardPath, indexMetaData, null, indexSearcherWrapper, engineFactory, globalCheckpointSyncer,
        EMPTY_EVENT_LISTENER, listeners);
}
 

@Inject
public NodeSysExpression(ClusterService clusterService,
                         OsService osService,
                         NodeService nodeService,
                         JvmService jvmService,
                         NodeEnvironment nodeEnvironment,
                         Discovery discovery,
                         ThreadPool threadPool,
                         ExtendedNodeInfo extendedNodeInfo) {
    this.nodeService = nodeService;
    this.osService = osService;
    this.jvmService = jvmService;
    this.nodeEnvironment = nodeEnvironment;
    this.extendedNodeInfo = extendedNodeInfo;
    childImplementations.put(SysNodesTableInfo.SYS_COL_HOSTNAME,
            new NodeHostnameExpression(clusterService));
    childImplementations.put(SysNodesTableInfo.SYS_COL_REST_URL,
            new NodeRestUrlExpression(clusterService));
    childImplementations.put(SysNodesTableInfo.SYS_COL_ID,
            new NodeIdExpression(clusterService));
    childImplementations.put(SysNodesTableInfo.SYS_COL_NODE_NAME,
            new NodeNameExpression(discovery));
    childImplementations.put(SysNodesTableInfo.SYS_COL_PORT,
            new NodePortExpression(nodeService));
    childImplementations.put(SysNodesTableInfo.SYS_COL_VERSION,
            new NodeVersionExpression());
    childImplementations.put(SysNodesTableInfo.SYS_COL_THREAD_POOLS,
            new NodeThreadPoolsExpression(threadPool));
    childImplementations.put(SysNodesTableInfo.SYS_COL_OS_INFO,
            new NodeOsInfoExpression(osService.info()));
}
 

/**
 * Returns <code>true</code> iff the given location contains an index an the index
 * can be successfully opened. This includes reading the segment infos and possible
 * corruption markers.
 */
public static boolean canOpenIndex(Logger logger, Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker) throws IOException {
    try {
        tryOpenIndex(indexLocation, shardId, shardLocker, logger);
    } catch (Exception ex) {
        logger.trace(() -> new ParameterizedMessage("Can't open index for path [{}]", indexLocation), ex);
        return false;
    }
    return true;
}
 

@After
public void closeNodeEnvironmentsAfterEachTest() {
    for (NodeEnvironment nodeEnvironment : nodeEnvironments) {
        nodeEnvironment.close();
    }
    nodeEnvironments.clear();
}
 

private Settings getNodeSettings(final int nodeId, final long seed, final Settings extraSettings, final int defaultMinMasterNodes) {
    final Settings settings = getSettings(nodeId, seed, extraSettings);

    final String name = buildNodeName(nodeId, settings);

    final Settings.Builder updatedSettings = Settings.builder();

    updatedSettings.put(Environment.PATH_HOME_SETTING.getKey(), baseDir);

    if (numDataPaths > 1) {
        updatedSettings.putList(Environment.PATH_DATA_SETTING.getKey(), IntStream.range(0, numDataPaths).mapToObj(i ->
            baseDir.resolve(name).resolve("d" + i).toString()).collect(Collectors.toList()));
    } else {
        updatedSettings.put(Environment.PATH_DATA_SETTING.getKey(), baseDir.resolve(name));
    }

    updatedSettings.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), baseDir.resolve(name + "-shared"));

    // allow overriding the above
    updatedSettings.put(settings);
    // force certain settings
    updatedSettings.put("node.name", name);
    updatedSettings.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);

    if (autoManageMinMasterNodes) {
        assertThat("if master nodes are automatically managed then nodes must complete a join cycle when starting",
            updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()), nullValue());
    }

    return updatedSettings.build();
}
 

private void clearDataIfNeeded(RestartCallback callback) throws IOException {
    if (callback.clearData(name)) {
        NodeEnvironment nodeEnv = node.getNodeEnvironment();
        if (nodeEnv.hasNodeFile()) {
            final Path[] locations = nodeEnv.nodeDataPaths();
            logger.debug("removing node data paths: [{}]", Arrays.toString(locations));
            IOUtils.rm(locations);
        }
    }
}
 

/**
 * This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple
 * directories with a valid shard state exist the one with the highest version will be used.
 * <b>Note:</b> this method resolves custom data locations for the shard.
 */
public static ShardPath loadShardPath(Logger logger, ShardId shardId, IndexSettings indexSettings, Path[] availableShardPaths,
                                       int nodeLockId, Path sharedDataPath) throws IOException {
    final String indexUUID = indexSettings.getUUID();
    Path loadedPath = null;
    for (Path path : availableShardPaths) {
        // EMPTY is safe here because we never call namedObject
        ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
        if (load != null) {
            if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
                logger.warn("{} found shard on path: [{}] with a different index UUID - this "
                    + "shard seems to be leftover from a different index with the same name. "
                    + "Remove the leftover shard in order to reuse the path with the current index", shardId, path);
                throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID
                    + " expected: " + indexUUID + " on shard path: " + path);
            }
            if (loadedPath == null) {
                loadedPath = path;
            } else {
                throw new IllegalStateException(shardId + " more than one shard state found");
            }
        }

    }
    if (loadedPath == null) {
        return null;
    } else {
        final Path dataPath;
        final Path statePath = loadedPath;
        if (indexSettings.hasCustomDataPath()) {
            dataPath = NodeEnvironment.resolveCustomLocation(indexSettings, shardId, sharedDataPath, nodeLockId);
        } else {
            dataPath = statePath;
        }
        logger.debug("{} loaded data path [{}], state path [{}]", shardId, dataPath, statePath);
        return new ShardPath(indexSettings.hasCustomDataPath(), dataPath, statePath, shardId);
    }
}
 

/**
 * This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple
 * directories with a valid shard state exist the one with the highest version will be used.
 * <b>Note:</b> this method resolves custom data locations for the shard.
 */
public static ShardPath loadShardPath(Logger logger, NodeEnvironment env, ShardId shardId, IndexSettings indexSettings) throws IOException {
    final Path[] paths = env.availableShardPaths(shardId);
    final int nodeLockId = env.getNodeLockId();
    final Path sharedDataPath = env.sharedDataPath();
    return loadShardPath(logger, shardId, indexSettings, paths, nodeLockId, sharedDataPath);
}
 

@Inject
public TransportExportAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                             TransportService transportService, IndicesService indicesService,
                             ScriptService scriptService, CacheRecycler cacheRecycler,
                             ExportParser exportParser, Exporter exporter, NodeEnvironment nodeEnv) {
    super(settings, threadPool, clusterService, transportService, indicesService, scriptService,
        cacheRecycler, exportParser, exporter, nodeEnv);
}
 

public FsService(final Settings settings, final NodeEnvironment nodeEnvironment, ClusterInfoService clusterInfoService) {
    this.probe = new FsProbe(nodeEnvironment);
    this.clusterInfoService = clusterInfoService;
    refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
    LOGGER.debug("using refresh_interval [{}]", refreshInterval);
    cache = new FsInfoCache(refreshInterval, stats(probe, null, LOGGER, null));
}
 

@Override
public Collection<Object> createComponents(Client client,
                                           ClusterService clusterService,
                                           ThreadPool threadPool,
                                           NamedXContentRegistry xContentRegistry,
                                           Environment environment,
                                           NodeEnvironment nodeEnvironment,
                                           NamedWriteableRegistry namedWriteableRegistry) {
    if (AzureConfiguration.isDiscoveryReady(settings, logger)) {
        return Collections.singletonList(azureComputeService());
    }
    return Collections.emptyList();
}