org.elasticsearch.indices.breaker.CircuitBreakerService Java Examples

/**
 * Create a circuit breaker that will break if the number of estimated
 * bytes grows above the limit. All estimations will be multiplied by
 * the given overheadConstant. Uses the given oldBreaker to initialize
 * the starting offset.
 * @param settings settings to configure this breaker
 * @param parent parent circuit breaker service to delegate tripped breakers to
 * @param name the name of the breaker
 * @param oldBreaker the previous circuit breaker to inherit the used value from (starting offset)
 */
public ChildMemoryCircuitBreaker(BreakerSettings settings,
                                 ChildMemoryCircuitBreaker oldBreaker,
                                 Logger logger,
                                 CircuitBreakerService parent,
                                 String name) {
    this.name = name;
    this.settings = settings;
    this.memoryBytesLimit = settings.getLimit();
    this.overheadConstant = settings.getOverhead();
    if (oldBreaker == null) {
        this.used = new AtomicLong(0);
        this.trippedCount = new AtomicLong(0);
    } else {
        this.used = oldBreaker.used;
        this.trippedCount = oldBreaker.trippedCount;
    }
    this.logger = logger;
    if (logger.isTraceEnabled()) {
        logger.trace("creating ChildCircuitBreaker with settings {}", this.settings);
    }
    this.parent = parent;
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore Circuit Breaker
    final Names fieldNames = fieldType.names();
    final Settings fdSettings = fieldType.fieldDataType().getSettings();
    final Map<String, Settings> filter = fdSettings.getGroups("filter");
    if (filter != null && !filter.isEmpty()) {
        throw new IllegalArgumentException("Doc values field data doesn't support filters [" + fieldNames.fullName() + "]");
    }

    if (BINARY_INDEX_FIELD_NAMES.contains(fieldNames.indexName())) {
        assert numericType == null;
        return new BinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType());
    } else if (numericType != null) {
        if (TimestampFieldMapper.NAME.equals(fieldNames.indexName())
                || Version.indexCreated(indexSettings).onOrAfter(Version.V_1_4_0_Beta1)) {
            return new SortedNumericDVIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType());
        } else {
            // prior to ES 1.4: multi-valued numerics were boxed inside a byte[] as BINARY
            return new BinaryDVNumericIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType());
        }
    } else {
        return new SortedSetDVOrdinalsIndexFieldData(index, cache, indexSettings, fieldNames, breakerService, fieldType.fieldDataType());
    }
}
 

public MockTcpTransport(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
                        CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
                        NetworkService networkService, Version mockVersion) {
    super("mock-tcp-transport",
          settings,
          threadPool,
          bigArrays,
          circuitBreakerService,
          namedWriteableRegistry,
          networkService);
    // we have our own crazy cached threadpool this one is not bounded at all...
    // using the ES thread factory here is crucial for tests otherwise disruption tests won't block that thread
    executor = Executors.newCachedThreadPool(EsExecutors.daemonThreadFactory(settings,
                                                                             Transports.TEST_MOCK_TRANSPORT_THREAD_PREFIX));
    this.mockVersion = mockVersion;
}
 

@Inject
public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService, Discovery discovery,
                   TransportService transportService, IndicesService indicesService,
                   PluginsService pluginService, CircuitBreakerService circuitBreakerService,
                   Version version) {
    super(settings);
    this.threadPool = threadPool;
    this.monitorService = monitorService;
    this.transportService = transportService;
    this.indicesService = indicesService;
    this.discovery = discovery;
    discovery.setNodeService(this);
    this.version = version;
    this.pluginService = pluginService;
    this.circuitBreakerService = circuitBreakerService;
}
 

/**
 * Constructor
 */
@Inject
public TransportTermsByQueryAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                   TransportService transportService, IndicesService indicesService,
                                   CircuitBreakerService breakerService,
                                   ScriptService scriptService, PageCacheRecycler pageCacheRecycler,
                                   BigArrays bigArrays, ActionFilters actionFilters,
                                   IndexNameExpressionResolver indexNameExpressionResolver, Client client) {
  super(settings, TermsByQueryAction.NAME, threadPool, clusterService, transportService, actionFilters,
          indexNameExpressionResolver, TermsByQueryRequest.class, TermsByQueryShardRequest.class,
          // Use the generic threadpool which is cached, as we can end up with deadlock with the SEARCH threadpool
          ThreadPool.Names.GENERIC);
  this.indicesService = indicesService;
  this.scriptService = scriptService;
  this.pageCacheRecycler = pageCacheRecycler;
  this.bigArrays = bigArrays;
  this.breakerService = breakerService;
  this.client = client;
}
 

@Test
public void testQueryCircuitBreakerDynamicSettings() throws Exception {
    CircuitBreakerService breakerService = new HierarchyCircuitBreakerService(
        Settings.EMPTY, clusterSettings);

    Settings newSettings = Settings.builder()
        .put(HierarchyCircuitBreakerService.QUERY_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 2.0)
        .build();

    clusterSettings.applySettings(newSettings);

    CircuitBreaker breaker = breakerService.getBreaker(HierarchyCircuitBreakerService.QUERY);
    assertThat(breaker, notNullValue());
    assertThat(breaker, instanceOf(CircuitBreaker.class));
    assertThat(breaker.getOverhead(), is(2.0));
}
 

@Inject
public RestSQLAction(Settings settings,
                     SQLOperations sqlOperations,
                     PipelineRegistry pipelineRegistry,
                     Provider<UserManager> userManagerProvider,
                     CircuitBreakerService breakerService,
                     SslContextProvider sslContextProvider) {
    UserManager userManager = userManagerProvider.get();
    pipelineRegistry.addBefore(new PipelineRegistry.ChannelPipelineItem(
        "handler",
        "sql_handler",
        corsConfig -> new SqlHttpHandler(
            settings,
            sqlOperations,
            breakerService::getBreaker,
            userManager,
            userManager::getAccessControl,
            corsConfig
        )
    ));
    pipelineRegistry.setSslContextProvider(sslContextProvider);
}
 

@Test
@UseHashJoins(1)
public void testInnerEquiJoinUsingHashJoin() {
    execute("create table t1 (a integer)");
    execute("create table t2 (x integer)");
    ensureYellow();
    execute("insert into t1 (a) values (0), (0), (1), (2), (4)");
    execute("insert into t2 (x) values (1), (3), (3), (4), (4)");
    execute("refresh table t1, t2");

    long memoryLimit = 6 * 1024 * 1024;
    double overhead = 1.0d;
    execute("set global \"indices.breaker.query.limit\" = '" + memoryLimit + "b', " +
            "\"indices.breaker.query.overhead\" = " + overhead);
    CircuitBreaker queryCircuitBreaker = internalCluster().getInstance(CircuitBreakerService.class).getBreaker(HierarchyCircuitBreakerService.QUERY);
    randomiseAndConfigureJoinBlockSize("t1", 5L, queryCircuitBreaker);
    randomiseAndConfigureJoinBlockSize("t2", 5L, queryCircuitBreaker);

    execute("select a, x from t1 join t2 on t1.a + 1 = t2.x order by a, x");
    assertThat(TestingHelpers.printedTable(response.rows()),
        is("0| 1\n" +
           "0| 1\n" +
           "2| 3\n" +
           "2| 3\n"));
}
 

@Override
public Map<String, Supplier<Transport>> getTransports(Settings settings,
                                                      ThreadPool threadPool,
                                                      BigArrays bigArrays,
                                                      PageCacheRecycler pageCacheRecycler,
                                                      CircuitBreakerService circuitBreakerService,
                                                      NamedWriteableRegistry namedWriteableRegistry,
                                                      NetworkService networkService) {
    return Collections.singletonMap(
        NETTY_TRANSPORT_NAME,
        () -> new Netty4Transport(
            settings,
            threadPool,
            networkService,
            bigArrays,
            namedWriteableRegistry,
            circuitBreakerService
        )
    );
}
 

@Override
public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
                                                                    CircuitBreakerService circuitBreakerService,
                                                                    NamedWriteableRegistry namedWriteableRegistry,
                                                                    NamedXContentRegistry xContentRegistry,
                                                                    NetworkService networkService,
                                                                    NodeClient nodeClient) {
    return Collections.singletonMap(
        NETTY_HTTP_TRANSPORT_NAME,
        () -> new Netty4HttpServerTransport(
            settings,
            networkService,
            bigArrays,
            threadPool,
            xContentRegistry,
            pipelineRegistry,
            nodeClient
        )
    );
}
 

@Inject
public CrateMonitor(JobsLogs jobsLogs,
                    PostgresNetty postgresNetty,
                    @Nullable HttpServerTransport httpServerTransport,
                    TransportService transportService,
                    SQLOperations sqlOperations,
                    ClusterService clusterService,
                    ThreadPool threadPool,
                    CircuitBreakerService breakerService) {
    logger = LogManager.getLogger(CrateMonitor.class);
    registerMBean(QueryStats.NAME, new QueryStats(jobsLogs));
    registerMBean(NodeStatus.NAME, new NodeStatus(sqlOperations::isEnabled));
    registerMBean(NodeInfo.NAME, new NodeInfo(clusterService::localNode, () -> clusterService.state().version()));
    registerMBean(Connections.NAME, new Connections(
        () -> httpServerTransport == null ? null : httpServerTransport.stats(),
        () -> new ConnectionStats(postgresNetty.openConnections(), postgresNetty.totalConnections()),
        () -> transportService.stats().serverOpen()
    ));
    registerMBean(ThreadPools.NAME, new ThreadPools(threadPool));
    registerMBean(CircuitBreakers.NAME, new CircuitBreakers(breakerService));
}
 

@Override
public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
        CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
        NamedXContentRegistry xContentRegistry, NetworkService networkService, Dispatcher dispatcher) {
    
    final Map<String, Supplier<HttpServerTransport>> httpTransports = new HashMap<String, Supplier<HttpServerTransport>>(1);
    if (!client && httpSSLEnabled) {
        
        final ValidatingDispatcher validatingDispatcher = new ValidatingDispatcher(threadPool.getThreadContext(), dispatcher, settings, configPath, NOOP_SSL_EXCEPTION_HANDLER);
        final OpenDistroSecuritySSLNettyHttpServerTransport sgsnht = new OpenDistroSecuritySSLNettyHttpServerTransport(settings, networkService, bigArrays, threadPool, odsks, xContentRegistry, validatingDispatcher, NOOP_SSL_EXCEPTION_HANDLER);
        
        httpTransports.put("com.amazon.opendistroforelasticsearch.security.ssl.http.netty.OpenDistroSecuritySSLNettyHttpServerTransport", () -> sgsnht);
        
    }
    return httpTransports;
}
 

@Inject
public TransportFetchNodeAction(TransportService transportService,
                                Transports transports,
                                ThreadPool threadPool,
                                StatsTables statsTables,
                                CircuitBreakerService breakerService,
                                JobContextService jobContextService,
                                NodeFetchOperation nodeFetchOperation) {
    this.transports = transports;
    this.statsTables = statsTables;
    this.nodeFetchOperation = nodeFetchOperation;
    this.circuitBreaker = breakerService.getBreaker(CrateCircuitBreakerService.QUERY);
    this.jobContextService = jobContextService;
    this.threadPool = threadPool;

    transportService.registerRequestHandler(TRANSPORT_ACTION,
            NodeFetchRequest.class,
            EXECUTOR_NAME,
            new NodeActionRequestHandler<NodeFetchRequest, NodeFetchResponse>(this) { });
}
 

public ParentChildIndexFieldData(Index index, Settings indexSettings, MappedFieldType.Names fieldNames,
                                 FieldDataType fieldDataType, IndexFieldDataCache cache, MapperService mapperService,
                                 CircuitBreakerService breakerService) {
    super(index, indexSettings, fieldNames, fieldDataType, cache);
    this.breakerService = breakerService;
    if (Version.indexCreated(indexSettings).before(Version.V_2_0_0_beta1)) {
        parentTypes = new TreeSet<>();
        for (DocumentMapper documentMapper : mapperService.docMappers(false)) {
            beforeCreate(documentMapper);
        }
        mapperService.addTypeListener(this);
    } else {
        ImmutableSortedSet.Builder<String> builder = ImmutableSortedSet.naturalOrder();
        for (DocumentMapper mapper : mapperService.docMappers(false)) {
            ParentFieldMapper parentFieldMapper = mapper.parentFieldMapper();
            if (parentFieldMapper.active()) {
                builder.add(parentFieldMapper.type());
            }
        }
        parentTypes = builder.build();
    }
}
 

@Inject
public CrateCircuitBreakerService(Settings settings,
                                  NodeSettingsService nodeSettingsService,
                                  CircuitBreakerService esCircuitBreakerService) {
    super(settings);
    this.esCircuitBreakerService = esCircuitBreakerService;

    long memoryLimit = settings.getAsMemory(
            QUERY_CIRCUIT_BREAKER_LIMIT_SETTING,
            DEFAULT_QUERY_CIRCUIT_BREAKER_LIMIT).bytes();
    double overhead = settings.getAsDouble(
            QUERY_CIRCUIT_BREAKER_OVERHEAD_SETTING,
            DEFAULT_QUERY_CIRCUIT_BREAKER_OVERHEAD_CONSTANT);

    queryBreakerSettings = new BreakerSettings(QUERY, memoryLimit, overhead,
            CircuitBreaker.Type.parseValue(
                    settings.get(QUERY_CIRCUIT_BREAKER_TYPE_SETTING,
                    DEFAULT_QUERY_CIRCUIT_BREAKER_TYPE)));

    registerBreaker(queryBreakerSettings);
    nodeSettingsService.addListener(new ApplySettings());
}
 

@Override
public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool,
        BigArrays bigArrays, CircuitBreakerService circuitBreakerService,
        NamedWriteableRegistry namedWriteableRegistry, NamedXContentRegistry namedXContentRegistry,
        NetworkService networkService, Dispatcher dispatcher) {
    Map<String, Supplier<HttpServerTransport>> transports = sgPlugin.getHttpTransports(settings, threadPool, bigArrays, circuitBreakerService,
            namedWriteableRegistry, namedXContentRegistry, networkService, dispatcher);
    return transports;
}
 

public OpenDistroSecuritySSLNettyTransport(final Settings settings, final Version version, final ThreadPool threadPool, final NetworkService networkService,
    final PageCacheRecycler pageCacheRecycler, final NamedWriteableRegistry namedWriteableRegistry,
    final CircuitBreakerService circuitBreakerService, final OpenDistroSecurityKeyStore odks, final SslExceptionHandler errorHandler) {
    super(settings, version, threadPool, networkService, pageCacheRecycler, namedWriteableRegistry, circuitBreakerService);
    this.odks = odks;
    this.errorHandler = errorHandler;
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType,
                               IndexFieldDataCache cache, CircuitBreakerService breakerService,
                               MapperService mapperService) {
    return new ParentChildIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache,
        mapperService, breakerService);
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore breaker
    return new GeoPointDVIndexFieldData(index, fieldType.names(), fieldType.fieldDataType(),
            Version.indexCreated(indexSettings).before(Version.V_2_2_0));
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore breaker
    final Names fieldNames = fieldType.names();
    return new BytesBinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType());
}
 

@Inject
public IndexFieldDataService(Index index, IndexSettingsService indexSettingsService, IndicesFieldDataCache indicesFieldDataCache,
                             CircuitBreakerService circuitBreakerService, MapperService mapperService) {
    super(index, indexSettingsService.getSettings());
    this.indicesFieldDataCache = indicesFieldDataCache;
    this.circuitBreakerService = circuitBreakerService;
    this.mapperService = mapperService;
}
 

public BigArrays(PageCacheRecycler recycler, @Nullable final CircuitBreakerService breakerService, boolean checkBreaker) {
    this.checkBreaker = checkBreaker;
    this.recycler = recycler;
    this.breakerService = breakerService;
    if (checkBreaker) {
        this.circuitBreakingInstance = this;
    } else {
        this.circuitBreakingInstance = new BigArrays(recycler, breakerService, true);
    }
}
 

public BigArrays(PageCacheRecycler recycler,
                 @Nullable final CircuitBreakerService breakerService,
                 String breakerName,
                 boolean checkBreaker) {
    this.checkBreaker = checkBreaker;
    this.recycler = recycler;
    this.breakerService = breakerService;
    this.breakerName = breakerName;
}
 

/**
 * Returns a map of {@link Transport} suppliers.
 * See {@link org.elasticsearch.common.network.NetworkModule#TRANSPORT_TYPE_KEY} to configure a specific implementation.
 */
default Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
                                                       PageCacheRecycler pageCacheRecycler,
                                                       CircuitBreakerService circuitBreakerService,
                                                       NamedWriteableRegistry namedWriteableRegistry,
                                                       NetworkService networkService) {
    return Collections.emptyMap();
}
 

@Override
public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
        CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
        NetworkService networkService) {
    Map<String, Supplier<Transport>> transports = sgPlugin.getTransports(settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry,
            networkService);
    return transports;
}
 

@Inject
public ReservoirSampler(ClusterService clusterService,
                        Functions functions,
                        Schemas schemas,
                        CircuitBreakerService circuitBreakerService,
                        IndicesService indicesService) {
    this.clusterService = clusterService;
    this.functions = functions;
    this.schemas = schemas;
    this.circuitBreakerService = circuitBreakerService;
    this.indicesService = indicesService;
}
 

@Inject
public TransportFetchNodeAction(Settings settings,
                                TransportService transportService,
                                Transports transports,
                                ThreadPool threadPool,
                                JobsLogs jobsLogs,
                                TasksService tasksService,
                                CircuitBreakerService circuitBreakerService) {
    this.transports = transports;
    this.nodeFetchOperation = new NodeFetchOperation(
        (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH),
        EsExecutors.numberOfProcessors(settings),
        jobsLogs,
        tasksService,
        circuitBreakerService.getBreaker(HierarchyCircuitBreakerService.QUERY)
    );

    transportService.registerRequestHandler(
        TRANSPORT_ACTION,
        NodeFetchRequest::new,
        EXECUTOR_NAME,
        // force execution because this handler might receive empty close requests which
        // need to be processed to not leak the FetchTask.
        // This shouldn't cause too much of an issue because fetch requests always happen after a query phase.
        // If the threadPool is overloaded the query phase would fail first.
        true,
        false,
        new NodeActionRequestHandler<>(this)
    );
}
 

@Inject
public JobsLogService(Settings settings,
                      ClusterService clusterService,
                      ClusterSettings clusterSettings,
                      Functions functions,
                      CircuitBreakerService breakerService) {
    this(
        settings,
        clusterService::localNode,
        clusterSettings,
        functions,
        Executors.newSingleThreadScheduledExecutor(),
        breakerService
    );
}
 

/**
 * Returns a map of {@link HttpServerTransport} suppliers.
 * See {@link org.elasticsearch.common.network.NetworkModule#HTTP_TYPE_SETTING} to configure a specific implementation.
 */
default Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings,
                                                                     ThreadPool threadPool,
                                                                     BigArrays bigArrays,
                                                                     CircuitBreakerService circuitBreakerService,
                                                                     NamedWriteableRegistry namedWriteableRegistry,
                                                                     NamedXContentRegistry xContentRegistry,
                                                                     NetworkService networkService,
                                                                     NodeClient nodeClient) {
    return Collections.emptyMap();
}
 

public TcpTransport(String transportName, Settings settings, ThreadPool threadPool, BigArrays bigArrays,
                    CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
                    NetworkService networkService) {
    this.settings = settings;
    this.profileSettings = getProfileSettings(settings);
    this.threadPool = threadPool;
    this.bigArrays = bigArrays;
    this.circuitBreakerService = circuitBreakerService;
    this.namedWriteableRegistry = namedWriteableRegistry;
    this.compress = Transport.TRANSPORT_TCP_COMPRESS.get(settings);
    this.networkService = networkService;
    this.transportName = transportName;
    this.nodeName = Node.NODE_NAME_SETTING.get(settings);
    final Settings defaultFeatures = DEFAULT_FEATURES_SETTING.get(settings);
    if (defaultFeatures == null) {
        this.features = new String[0];
    } else {
        defaultFeatures.names().forEach(key -> {
            if (Booleans.parseBoolean(defaultFeatures.get(key)) == false) {
                throw new IllegalArgumentException("feature settings must have default [true] value");
            }
        });
        // use a sorted set to present the features in a consistent order
        this.features = new TreeSet<>(defaultFeatures.names()).toArray(new String[defaultFeatures.names().size()]);
    }

    try (BytesStreamOutput out = new BytesStreamOutput()) {
        out.writeByte((byte) 'E');
        out.writeByte((byte) 'S');
        out.writeInt(TcpTransport.PING_DATA_SIZE);
        pingMessage = out.bytes();
    } catch (IOException e) {
        throw new AssertionError(e.getMessage(), e); // won't happen
    }
}