Java Code Examples for org.elasticsearch.common.unit.ByteSizeValue#getBytes()

/**
 * Create a circuit breaker that will break if the number of estimated
 * bytes grows above the limit. All estimations will be multiplied by
 * the given overheadConstant. Uses the given oldBreaker to initialize
 * the starting offset.
 * @param limit circuit breaker limit
 * @param overheadConstant constant multiplier for byte estimations
 * @param oldBreaker the previous circuit breaker to inherit the used value from (starting offset)
 */
public MemoryCircuitBreaker(ByteSizeValue limit, double overheadConstant, MemoryCircuitBreaker oldBreaker, Logger logger) {
    this.memoryBytesLimit = limit.getBytes();
    this.overheadConstant = overheadConstant;
    if (oldBreaker == null) {
        this.used = new AtomicLong(0);
        this.trippedCount = new AtomicLong(0);
    } else {
        this.used = oldBreaker.used;
        this.trippedCount = oldBreaker.trippedCount;
    }
    this.logger = logger;
    if (logger.isTraceEnabled()) {
        logger.trace("Creating MemoryCircuitBreaker with a limit of {} bytes ({}) and a overhead constant of {}",
                this.memoryBytesLimit, limit, this.overheadConstant);
    }
}
 

private static void doValidateAsBytes(final String low, final String high, final String flood) {
    final ByteSizeValue lowWatermarkBytes =
            thresholdBytesFromWatermark(low, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), false);
    final ByteSizeValue highWatermarkBytes =
            thresholdBytesFromWatermark(high, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), false);
    final ByteSizeValue floodStageBytes =
            thresholdBytesFromWatermark(flood, CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), false);
    if (lowWatermarkBytes.getBytes() < highWatermarkBytes.getBytes()) {
        throw new IllegalArgumentException(
                "low disk watermark [" + low + "] less than high disk watermark [" + high + "]");
    }
    if (highWatermarkBytes.getBytes() < floodStageBytes.getBytes()) {
        throw new IllegalArgumentException(
                "high disk watermark [" + high + "] less than flood stage disk watermark [" + flood + "]");
    }
}
 

/**
 * Constructs a new instance of file info
 *
 * @param name         file name as stored in the blob store
 * @param metaData  the files meta data
 * @param partSize     size of the single chunk
 */
public FileInfo(String name, StoreFileMetaData metaData, ByteSizeValue partSize) {
    this.name = name;
    this.metadata = metaData;

    long partBytes = Long.MAX_VALUE;
    if (partSize != null && partSize.getBytes() > 0) {
        partBytes = partSize.getBytes();
    }

    long totalLength = metaData.length();
    long numberOfParts = totalLength / partBytes;
    if (totalLength % partBytes > 0) {
        numberOfParts++;
    }
    if (numberOfParts == 0) {
        numberOfParts++;
    }
    this.numberOfParts = numberOfParts;
    this.partSize = partSize;
    this.partBytes = partBytes;
}
 

IndexingMemoryController(Settings settings, ThreadPool threadPool, Iterable<IndexShard> indexServices) {
    this.indexShards = indexServices;

    ByteSizeValue indexingBuffer = INDEX_BUFFER_SIZE_SETTING.get(settings);

    String indexingBufferSetting = settings.get(INDEX_BUFFER_SIZE_SETTING.getKey());
    // null means we used the default (10%)
    if (indexingBufferSetting == null || indexingBufferSetting.endsWith("%")) {
        // We only apply the min/max when % value was used for the index buffer:
        ByteSizeValue minIndexingBuffer = MIN_INDEX_BUFFER_SIZE_SETTING.get(settings);
        ByteSizeValue maxIndexingBuffer = MAX_INDEX_BUFFER_SIZE_SETTING.get(settings);
        if (indexingBuffer.getBytes() < minIndexingBuffer.getBytes()) {
            indexingBuffer = minIndexingBuffer;
        }
        if (maxIndexingBuffer.getBytes() != -1 && indexingBuffer.getBytes() > maxIndexingBuffer.getBytes()) {
            indexingBuffer = maxIndexingBuffer;
        }
    }
    this.indexingBuffer = indexingBuffer;

    this.inactiveTime = SHARD_INACTIVE_TIME_SETTING.get(settings);
    // we need to have this relatively small to free up heap quickly enough
    this.interval = SHARD_MEMORY_INTERVAL_TIME_SETTING.get(settings);

    this.statusChecker = new ShardsIndicesStatusChecker();

    LOGGER.debug("using indexing buffer size [{}] with {} [{}], {} [{}]",
                 this.indexingBuffer,
                 SHARD_INACTIVE_TIME_SETTING.getKey(), this.inactiveTime,
                 SHARD_MEMORY_INTERVAL_TIME_SETTING.getKey(), this.interval);
    this.scheduler = scheduleTask(threadPool);

    // Need to save this so we can later launch async "write indexing buffer to disk" on shards:
    this.threadPool = threadPool;
}
 

private void configureSocket(Socket socket) throws SocketException {
    socket.setTcpNoDelay(TransportSettings.TCP_NO_DELAY.get(settings));
    ByteSizeValue tcpSendBufferSize = TransportSettings.TCP_SEND_BUFFER_SIZE.get(settings);
    if (tcpSendBufferSize.getBytes() > 0) {
        socket.setSendBufferSize(tcpSendBufferSize.bytesAsInt());
    }
    ByteSizeValue tcpReceiveBufferSize = TransportSettings.TCP_RECEIVE_BUFFER_SIZE.get(settings);
    if (tcpReceiveBufferSize.getBytes() > 0) {
        socket.setReceiveBufferSize(tcpReceiveBufferSize.bytesAsInt());
    }
    socket.setReuseAddress(TransportSettings.TCP_REUSE_ADDRESS.get(settings));
}
 

private Bootstrap createClientBootstrap() {
    final Bootstrap bootstrap = new Bootstrap();
    if (Epoll.isAvailable()) {
        bootstrap.group(new EpollEventLoopGroup(workerCount, daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)));
        bootstrap.channel(EpollSocketChannel.class);
    } else {
        bootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)));
        bootstrap.channel(NioSocketChannel.class);
    }

    bootstrap.option(ChannelOption.TCP_NODELAY, TransportSettings.TCP_NO_DELAY.get(settings));
    bootstrap.option(ChannelOption.SO_KEEPALIVE, TransportSettings.TCP_KEEP_ALIVE.get(settings));

    final ByteSizeValue tcpSendBufferSize = TransportSettings.TCP_SEND_BUFFER_SIZE.get(settings);
    if (tcpSendBufferSize.getBytes() > 0) {
        bootstrap.option(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.getBytes()));
    }

    final ByteSizeValue tcpReceiveBufferSize = TransportSettings.TCP_RECEIVE_BUFFER_SIZE.get(settings);
    if (tcpReceiveBufferSize.getBytes() > 0) {
        bootstrap.option(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.getBytes()));
    }

    bootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator);

    final boolean reuseAddress = TransportSettings.TCP_REUSE_ADDRESS.get(settings);
    bootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress);

    return bootstrap;
}
 

/**
 * Configures RateLimiter based on repository and global settings
 *
 * @param repositorySettings repository settings
 * @param setting            setting to use to configure rate limiter
 * @param defaultRate        default limiting rate
 * @return rate limiter or null of no throttling is needed
 */
private RateLimiter getRateLimiter(Settings repositorySettings, String setting, ByteSizeValue defaultRate) {
    ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.getAsBytesSize(setting,
            settings.getAsBytesSize(setting, defaultRate));
    if (maxSnapshotBytesPerSec.getBytes() <= 0) {
        return null;
    } else {
        return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.getMbFrac());
    }
}
 

@Override
protected void doStart() {
    ByteSizeValue chunkSize = chunkSize();
    if (chunkSize != null && chunkSize.getBytes() <= 0) {
        throw new IllegalArgumentException("the chunk size cannot be negative: [" + chunkSize + "]");
    }
}
 

public Caches(TimeValue expAfterWrite, TimeValue expAfterAccess, ByteSizeValue maxWeight) {
    this.featureCache = configCache(CacheBuilder.<CacheKey, Feature>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher(Caches::weigther)
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.featureSetCache = configCache(CacheBuilder.<CacheKey, FeatureSet>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher(Caches::weigther)
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.modelCache = configCache(CacheBuilder.<CacheKey, CompiledLtrModel>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher((s, w) -> w.ramBytesUsed())
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.maxWeight = maxWeight.getBytes();
}
 

public IndicesQueryCache(Settings settings) {
    final ByteSizeValue size = INDICES_CACHE_QUERY_SIZE_SETTING.get(settings);
    final int count = INDICES_CACHE_QUERY_COUNT_SETTING.get(settings);
    LOGGER.debug("using [node] query cache with size [{}] max filter count [{}]",
            size, count);
    if (INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.get(settings)) {
        cache = new ElasticsearchLRUQueryCache(count, size.getBytes(), context -> true, 1f);
    } else {
        cache = new ElasticsearchLRUQueryCache(count, size.getBytes());
    }
    sharedRamBytesUsed = 0;
}
 

private void setBreakerLimit(BreakerSettings oldSettings,
                             String breakerName,
                             Consumer<BreakerSettings> settingsConsumer,
                             ByteSizeValue newLimit, Double newOverhead) {
    long newLimitBytes = newLimit == null ? oldSettings.getLimit() : newLimit.getBytes();
    newOverhead = newOverhead == null ? oldSettings.getOverhead() : newOverhead;
    BreakerSettings newSettings = new BreakerSettings(breakerName, newLimitBytes, newOverhead, oldSettings.getType());
    registerBreaker(newSettings);
    settingsConsumer.accept(newSettings);
    LOGGER.info("[{}] Updated breaker settings: {}", breakerName, newSettings);
}
 

private void setAccountingBreakerLimit(ByteSizeValue newAccountingMax, Double newAccountingOverhead) {
    BreakerSettings newAccountingSettings = new BreakerSettings(CircuitBreaker.ACCOUNTING, newAccountingMax.getBytes(),
        newAccountingOverhead, HierarchyCircuitBreakerService.this.inFlightRequestsSettings.getType());
    registerBreaker(newAccountingSettings);
    HierarchyCircuitBreakerService.this.accountingSettings = newAccountingSettings;
    LOGGER.info("Updated breaker settings for accounting requests: {}", newAccountingSettings);
}
 

private void setFieldDataBreakerLimit(ByteSizeValue newFielddataMax, Double newFielddataOverhead) {
    long newFielddataLimitBytes = newFielddataMax == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getLimit() : newFielddataMax.getBytes();
    newFielddataOverhead = newFielddataOverhead == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getOverhead() : newFielddataOverhead;
    BreakerSettings newFielddataSettings = new BreakerSettings(CircuitBreaker.FIELDDATA, newFielddataLimitBytes, newFielddataOverhead,
            HierarchyCircuitBreakerService.this.fielddataSettings.getType());
    registerBreaker(newFielddataSettings);
    HierarchyCircuitBreakerService.this.fielddataSettings = newFielddataSettings;
    LOGGER.info("Updated breaker settings field data: {}", newFielddataSettings);
}
 

private void setInFlightRequestsBreakerLimit(ByteSizeValue newInFlightRequestsMax, Double newInFlightRequestsOverhead) {
    BreakerSettings newInFlightRequestsSettings = new BreakerSettings(CircuitBreaker.IN_FLIGHT_REQUESTS, newInFlightRequestsMax.getBytes(),
        newInFlightRequestsOverhead, HierarchyCircuitBreakerService.this.inFlightRequestsSettings.getType());
    registerBreaker(newInFlightRequestsSettings);
    HierarchyCircuitBreakerService.this.inFlightRequestsSettings = newInFlightRequestsSettings;
    LOGGER.info("Updated breaker settings for in-flight requests: {}", newInFlightRequestsSettings);
}
 

private void setRequestBreakerLimit(ByteSizeValue newRequestMax, Double newRequestOverhead) {
    BreakerSettings newRequestSettings = new BreakerSettings(CircuitBreaker.REQUEST, newRequestMax.getBytes(), newRequestOverhead,
            HierarchyCircuitBreakerService.this.requestSettings.getType());
    registerBreaker(newRequestSettings);
    HierarchyCircuitBreakerService.this.requestSettings = newRequestSettings;
    LOGGER.info("Updated breaker settings request: {}", newRequestSettings);
}
 

private void setTotalCircuitBreakerLimit(ByteSizeValue byteSizeValue) {
    BreakerSettings newParentSettings = new BreakerSettings(CircuitBreaker.PARENT, byteSizeValue.getBytes(), 1.0, CircuitBreaker.Type.PARENT);
    this.parentSettings = newParentSettings;
}
 

@Override
protected void doStart() {
    boolean success = false;
    try {
        this.serverOpenChannels = new Netty4OpenChannelsHandler(logger);

        serverBootstrap = new ServerBootstrap();

        ThreadFactory threadFactory = daemonThreadFactory(settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX);
        if (Epoll.isAvailable()) {
            serverBootstrap.group(new EpollEventLoopGroup(workerCount, threadFactory));
            serverBootstrap.channel(EpollServerSocketChannel.class);
        } else {
            serverBootstrap.group(new NioEventLoopGroup(workerCount, threadFactory));
            serverBootstrap.channel(NioServerSocketChannel.class);
        }

        serverBootstrap.childHandler(configureServerChannelHandler());

        serverBootstrap.childOption(ChannelOption.TCP_NODELAY, SETTING_HTTP_TCP_NO_DELAY.get(settings));
        serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, SETTING_HTTP_TCP_KEEP_ALIVE.get(settings));

        final ByteSizeValue tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings);
        if (tcpSendBufferSize.getBytes() > 0) {
            serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.getBytes()));
        }

        final ByteSizeValue tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings);
        if (tcpReceiveBufferSize.getBytes() > 0) {
            serverBootstrap.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.getBytes()));
        }

        serverBootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator);
        serverBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator);

        final boolean reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings);
        serverBootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress);
        serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress);

        this.boundAddress = createBoundHttpAddress();
        if (logger.isInfoEnabled()) {
            logger.info("{}", boundAddress);
        }
        success = true;
    } finally {
        if (success == false) {
            doStop(); // otherwise we leak threads since we never moved to started
        }
    }
}
 

private boolean validateTotalCircuitBreakerLimit(ByteSizeValue byteSizeValue) {
    BreakerSettings newParentSettings = new BreakerSettings(CircuitBreaker.PARENT, byteSizeValue.getBytes(), 1.0, CircuitBreaker.Type.PARENT);
    validateSettings(new BreakerSettings[]{newParentSettings});
    return true;
}