org.elasticsearch.common.breaker.CircuitBreaker Java Examples

The following examples show how to use org.elasticsearch.common.breaker.CircuitBreaker. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NestedLoopOperation.java    From crate with Apache License 2.0 6 votes vote down vote up
private static BatchIterator<Row> buildCrossJoinBatchIterator(BatchIterator<Row> left,
                                                              BatchIterator<Row> right,
                                                              CombinedRow combiner,
                                                              CircuitBreaker circuitBreaker,
                                                              RamAccounting ramAccounting,
                                                              List<DataType<?>> leftSideColumnTypes,
                                                              long estimatedRowsSizeLeft,
                                                              long estimatedNumberOfRowsLeft,
                                                              boolean blockNestedLoop) {
    if (blockNestedLoop) {
        IntSupplier blockSizeCalculator = new RamBlockSizeCalculator(
            Paging.PAGE_SIZE,
            circuitBreaker,
            estimatedRowsSizeLeft,
            estimatedNumberOfRowsLeft
        );
        var rowAccounting = new RowCellsAccountingWithEstimators(leftSideColumnTypes, ramAccounting, 0);
        return JoinBatchIterators.crossJoinBlockNL(left, right, combiner, blockSizeCalculator, rowAccounting);
    } else {
        return JoinBatchIterators.crossJoinNL(left, right, combiner);
    }
}
 
Example #2
Source File: JobSetup.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public Boolean visitCollectPhase(CollectPhase phase, Context context) {
    CircuitBreaker breaker = breaker();
    int ramAccountingBlockSizeInBytes = BlockBasedRamAccounting.blockSizeInBytes(breaker.getLimit());
    RamAccounting ramAccounting = ConcurrentRamAccounting.forCircuitBreaker(phase.label(), breaker);
    RowConsumer consumer = context.getRowConsumer(
        phase,
        Paging.PAGE_SIZE,
        new BlockBasedRamAccounting(ramAccounting::addBytes, ramAccountingBlockSizeInBytes)
    );
    consumer.completionFuture().whenComplete((result, error) -> ramAccounting.close());
    context.registerSubContext(new CollectTask(
        phase,
        context.txnCtx(),
        collectOperation,
        ramAccounting,
        memoryManagerFactory,
        consumer,
        context.sharedShardContexts,
        clusterService.state().getNodes().getMinNodeVersion(),
        ramAccountingBlockSizeInBytes
    ));
    return true;
}
 
Example #3
Source File: GeoPointArrayIndexFieldData.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception {
    LeafReader reader = context.reader();

    Terms terms = reader.terms(getFieldNames().indexName());
    AtomicGeoPointFieldData data = null;
    // TODO: Use an actual estimator to estimate before loading.
    NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker(CircuitBreaker.FIELDDATA));
    if (terms == null) {
        data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc());
        estimator.afterLoad(null, data.ramBytesUsed());
        return data;
    }
    return (Version.indexCreated(indexSettings).before(Version.V_2_2_0)) ?
        loadLegacyFieldData(reader, estimator, terms, data) : loadFieldData22(reader, estimator, terms, data);
}
 
Example #4
Source File: TermsSetTest.java    From siren-join with GNU Affero General Public License v3.0 6 votes vote down vote up
@Test
public void testCircuitBreakerAdjustmentOnIntTermsSet() {
  HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService(
          Settings.builder().build(),
          new NodeSettingsService(Settings.EMPTY));

  CircuitBreaker breaker = hcbs.getBreaker(CircuitBreaker.REQUEST);
  assertThat(breaker.getUsed(), is(equalTo(0L)));

  IntegerTermsSet termsSet = new IntegerTermsSet(8, hcbs.getBreaker(CircuitBreaker.REQUEST));
  long usedMem = breaker.getUsed();
  assertThat(usedMem, greaterThan(0L));

  for (int i = 0; i < 16; i++) {
    termsSet.add(i);
  }

  assertThat(breaker.getUsed(), greaterThan(usedMem));

  termsSet.release();

  assertThat(breaker.getUsed(), is(equalTo(0L)));
}
 
Example #5
Source File: CircuitBreakerTest.java    From siren-join with GNU Affero General Public License v3.0 6 votes vote down vote up
/** Reset all breaker settings back to their defaults */
@After
public void teardown() {
  logger.info("--> resetting breaker settings");
  Settings resetSettings = settingsBuilder()
          .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING,
                  HierarchyCircuitBreakerService.DEFAULT_FIELDDATA_BREAKER_LIMIT)
          .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING,
                  HierarchyCircuitBreakerService.DEFAULT_FIELDDATA_OVERHEAD_CONSTANT)
          .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING,
                  HierarchyCircuitBreakerService.DEFAULT_REQUEST_BREAKER_LIMIT)
          .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING, CircuitBreaker.Type.MEMORY)
          .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING, 1.0)
          .build();
  assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings));
}
 
Example #6
Source File: CircuitBreakerTest.java    From siren-join with GNU Affero General Public License v3.0 6 votes vote down vote up
@Test
public void testCircuitBreakerOnShard() throws Exception {
  // Update circuit breaker settings
  Settings settings = settingsBuilder()
          .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, "8b")
          .build();
  assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));

  SearchRequestBuilder searchRequest = new CoordinateSearchRequestBuilder(client()).setIndices("index1").setQuery(
          QueryBuilders.filterJoin("foreign_key").indices("index2").types("type").path("id").query(
                  boolQuery().filter(termQuery("tag", "aaa"))
          ).termsEncoding(TermsByQueryRequest.TermsEncoding.LONG)
  );
  assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR,
          containsString("Data too large, data for [<terms_set>] would be larger than limit of [8/8b]"));

  NodesStatsResponse stats = client().admin().cluster().prepareNodesStats().setBreaker(true).get();
  int breaks = 0;
  for (NodeStats stat : stats.getNodes()) {
    CircuitBreakerStats breakerStats = stat.getBreaker().getStats(CircuitBreaker.REQUEST);
    breaks += breakerStats.getTrippedCount();
  }
  assertThat(breaks, greaterThanOrEqualTo(1));
}
 
Example #7
Source File: CrateCircuitBreakerServiceTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testQueryCircuitBreakerDynamicSettings() throws Exception {
    CircuitBreakerService breakerService = new HierarchyCircuitBreakerService(
        Settings.EMPTY, clusterSettings);

    Settings newSettings = Settings.builder()
        .put(HierarchyCircuitBreakerService.QUERY_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 2.0)
        .build();

    clusterSettings.applySettings(newSettings);

    CircuitBreaker breaker = breakerService.getBreaker(HierarchyCircuitBreakerService.QUERY);
    assertThat(breaker, notNullValue());
    assertThat(breaker, instanceOf(CircuitBreaker.class));
    assertThat(breaker.getOverhead(), is(2.0));
}
 
Example #8
Source File: HierarchyCircuitBreakerService.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public CircuitBreakerStats stats(String name) {
    if (CircuitBreaker.PARENT.equals(name)) {
        return new CircuitBreakerStats(
            CircuitBreaker.PARENT,
            parentSettings.getLimit(),
            parentUsed(0L),
            parentTripCount.get(), 1.0
        );
    }
    CircuitBreaker breaker = requireNonNull(this.breakers.get(name), "Unknown circuit breaker: " + name);
    return new CircuitBreakerStats(
        breaker.getName(),
        breaker.getLimit(),
        breaker.getUsed(),
        breaker.getTrippedCount(),
        breaker.getOverhead());
}
 
Example #9
Source File: CrateCircuitBreakerService.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Inject
public CrateCircuitBreakerService(Settings settings,
                                  NodeSettingsService nodeSettingsService,
                                  CircuitBreakerService esCircuitBreakerService) {
    super(settings);
    this.esCircuitBreakerService = esCircuitBreakerService;

    long memoryLimit = settings.getAsMemory(
            QUERY_CIRCUIT_BREAKER_LIMIT_SETTING,
            DEFAULT_QUERY_CIRCUIT_BREAKER_LIMIT).bytes();
    double overhead = settings.getAsDouble(
            QUERY_CIRCUIT_BREAKER_OVERHEAD_SETTING,
            DEFAULT_QUERY_CIRCUIT_BREAKER_OVERHEAD_CONSTANT);

    queryBreakerSettings = new BreakerSettings(QUERY, memoryLimit, overhead,
            CircuitBreaker.Type.parseValue(
                    settings.get(QUERY_CIRCUIT_BREAKER_TYPE_SETTING,
                    DEFAULT_QUERY_CIRCUIT_BREAKER_TYPE)));

    registerBreaker(queryBreakerSettings);
    nodeSettingsService.addListener(new ApplySettings());
}
 
Example #10
Source File: TermsSetTest.java    From siren-join with GNU Affero General Public License v3.0 5 votes vote down vote up
@Test(expected=CircuitBreakingException.class)
public void testCircuitBreakerOnNewIntTermsSet() {
  final int size = 42;
  HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService(
          Settings.builder()
                  .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, size - 1, ByteSizeUnit.BYTES)
                  .build(),
          new NodeSettingsService(Settings.EMPTY));

  IntegerTermsSet termsSet = new IntegerTermsSet(size, hcbs.getBreaker(CircuitBreaker.REQUEST));
}
 
Example #11
Source File: BigArrays.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/**
 * Adjust the circuit breaker with the given delta, if the delta is
 * negative, or checkBreaker is false, the breaker will be adjusted
 * without tripping
 */
void adjustBreaker(long delta) {
    if (this.breakerService != null) {
        CircuitBreaker breaker = this.breakerService.getBreaker(CircuitBreaker.REQUEST);
        if (this.checkBreaker == true) {
            // checking breaker means potentially tripping, but it doesn't
            // have to if the delta is negative
            if (delta > 0) {
                try {
                    breaker.addEstimateBytesAndMaybeBreak(delta, "<reused_arrays>");
                } catch (CircuitBreakingException e) {
                    // since we've already created the data, we need to
                    // add it so closing the stream re-adjusts properly
                    breaker.addWithoutBreaking(delta);
                    // re-throw the original exception
                    throw e;
                }
            } else {
                breaker.addWithoutBreaking(delta);
            }
        } else {
            // even if we are not checking the breaker, we need to adjust
            // its' totals, so add without breaking
            breaker.addWithoutBreaking(delta);
        }
    }
}
 
Example #12
Source File: JoinIntegrationTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void test_block_NestedLoop_or_HashJoin__with_group_by_on_right_side() {
    execute("create table t1 (x integer)");
    execute("insert into t1 (x) values (0), (1), (2), (3), (4), (5), (6), (7), (8), (9)");
    execute("refresh table t1");

    long memoryLimit = 6 * 1024 * 1024;
    double overhead = 1.0d;
    execute("set global \"indices.breaker.query.limit\" = '" + memoryLimit + "b', " +
            "\"indices.breaker.query.overhead\" = " + overhead);
    CircuitBreaker queryCircuitBreaker = internalCluster().getInstance(CircuitBreakerService.class).getBreaker(HierarchyCircuitBreakerService.QUERY);
    randomiseAndConfigureJoinBlockSize("t1", 10L, queryCircuitBreaker);

    execute("select x from t1 left_rel JOIN (select x x2, count(x) from t1 group by x2) right_rel " +
            "ON left_rel.x = right_rel.x2 order by left_rel.x");

    assertThat(TestingHelpers.printedTable(response.rows()),
        is("0\n" +
           "1\n" +
           "2\n" +
           "3\n" +
           "4\n" +
           "5\n" +
           "6\n" +
           "7\n" +
           "8\n" +
           "9\n"));
}
 
Example #13
Source File: ParentChildIndexFieldData.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
    final long startTime = System.nanoTime();
    final Set<String> parentTypes;
    if (Version.indexCreated(indexSettings()).before(Version.V_2_0_0_beta1)) {
        synchronized (lock) {
            parentTypes = ImmutableSet.copyOf(this.parentTypes);
        }
    } else {
        parentTypes = this.parentTypes;
    }

    long ramBytesUsed = 0;
    final Map<String, OrdinalMapAndAtomicFieldData> perType = new HashMap<>();
    for (String type : parentTypes) {
        final AtomicParentChildFieldData[] fieldData = new AtomicParentChildFieldData[indexReader.leaves().size()];
        for (LeafReaderContext context : indexReader.leaves()) {
            fieldData[context.ord] = load(context);
        }
        final OrdinalMap ordMap = buildOrdinalMap(fieldData, type);
        ramBytesUsed += ordMap.ramBytesUsed();
        perType.put(type, new OrdinalMapAndAtomicFieldData(ordMap, fieldData));
    }

    final AtomicParentChildFieldData[] fielddata = new AtomicParentChildFieldData[indexReader.leaves().size()];
    for (int i = 0; i < fielddata.length; ++i) {
        fielddata[i] = new GlobalAtomicFieldData(parentTypes, perType, i);
    }

    breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(ramBytesUsed);
    if (logger.isDebugEnabled()) {
        logger.debug(
                "Global-ordinals[_parent] took {}",
                new TimeValue(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)
        );
    }

    return new GlobalFieldData(indexReader, fielddata, ramBytesUsed, perType);
}
 
Example #14
Source File: SqlHttpHandler.java    From crate with Apache License 2.0 5 votes vote down vote up
SqlHttpHandler(Settings settings,
               SQLOperations sqlOperations,
               Function<String, CircuitBreaker> circuitBreakerProvider,
               UserLookup userLookup,
               Function<SessionContext, AccessControl> getAccessControl,
               Netty4CorsConfig corsConfig) {
    super(false);
    this.settings = settings;
    this.sqlOperations = sqlOperations;
    this.circuitBreakerProvider = circuitBreakerProvider;
    this.userLookup = userLookup;
    this.getAccessControl = getAccessControl;
    this.corsConfig = corsConfig;
}
 
Example #15
Source File: RamAccountingTermsEnum.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public RamAccountingTermsEnum(TermsEnum termsEnum, CircuitBreaker breaker, AbstractIndexFieldData.PerValueEstimator estimator,
                              String fieldName) {
    super(termsEnum);
    this.breaker = breaker;
    this.termsEnum = termsEnum;
    this.estimator = estimator;
    this.fieldName = fieldName;
    this.totalBytes = 0;
    this.flushBuffer = 0;
}
 
Example #16
Source File: CircuitBreakerIntegrationTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testQueryBreakerIsDecrementedWhenQueryCompletes() {
    execute("create table t1 (text string)");
    execute("insert into t1 values ('this is some text'), ('other text')");
    refresh();

    CircuitBreakerService circuitBreakerService = internalCluster().getInstance(CircuitBreakerService.class);
    CircuitBreaker queryBreaker = circuitBreakerService.getBreaker(HierarchyCircuitBreakerService.QUERY);
    long breakerBytesUsedBeforeQuery = queryBreaker.getUsed();

    execute("select text from t1 group by text");

    assertThat(queryBreaker.getUsed(), is(breakerBytesUsedBeforeQuery));
}
 
Example #17
Source File: CrateCircuitBreakerServiceTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testQueryCircuitBreakerRegistration() throws Exception {
    CircuitBreakerService breakerService = new HierarchyCircuitBreakerService(
        Settings.EMPTY,
        clusterSettings
    );

    CircuitBreaker breaker = breakerService.getBreaker(HierarchyCircuitBreakerService.QUERY);
    assertThat(breaker, notNullValue());
    assertThat(breaker, instanceOf(CircuitBreaker.class));
    assertThat(breaker.getName(), is(HierarchyCircuitBreakerService.QUERY));
}
 
Example #18
Source File: TermsSet.java    From siren-join with GNU Affero General Public License v3.0 5 votes vote down vote up
/**
 * Used by {@link solutions.siren.join.action.terms.TransportTermsByQueryAction}
 */
public static TermsSet newTermsSet(long expectedElements, TermsByQueryRequest.TermsEncoding termsEncoding, CircuitBreaker breaker) {
  switch (termsEncoding) {
    case LONG:
      return new LongTermsSet(expectedElements, breaker);
    case INTEGER:
      return new IntegerTermsSet(expectedElements, breaker);
    case BLOOM:
      return new BloomFilterTermsSet(expectedElements, breaker);
    case BYTES:
      return new BytesRefTermsSet(breaker);
    default:
      throw new IllegalArgumentException("[termsByQuery] Invalid terms encoding: " + termsEncoding.name());
  }
}
 
Example #19
Source File: TermsSetTest.java    From siren-join with GNU Affero General Public License v3.0 5 votes vote down vote up
@Test(expected=CircuitBreakingException.class)
public void testCircuitBreakerOnNewLongTermsSet() {
  final int size = 42;
  HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService(
          Settings.builder()
                  .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, size - 1, ByteSizeUnit.BYTES)
                  .build(),
          new NodeSettingsService(Settings.EMPTY));

  LongTermsSet termsSet = new LongTermsSet(size, hcbs.getBreaker(CircuitBreaker.REQUEST));
}
 
Example #20
Source File: HierarchyCircuitBreakerService.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Checks whether the parent breaker has been tripped
 */
public void checkParentLimit(long newBytesReserved, String label) throws CircuitBreakingException {
    long totalUsed = parentUsed(newBytesReserved);
    long parentLimit = this.parentSettings.getLimit();
    if (totalUsed > parentLimit) {
        long breakersTotalUsed = breakers.values().stream()
            .mapToLong(x -> (long) (x.getUsed() * x.getOverhead()))
            .sum();
        // if the individual breakers hardly use any memory we assume that there is a lot of heap usage by objects which can be GCd.
        // We want to allow the query so that it triggers GCs
        if ((breakersTotalUsed + newBytesReserved) < (parentLimit * PARENT_BREAKER_ESCAPE_HATCH_PERCENTAGE)) {
            return;
        }
        this.parentTripCount.incrementAndGet();
        final StringBuilder message = new StringBuilder(
            "[parent] Data too large, data for [" + label + "]" +
            " would be [" + totalUsed + "/" + new ByteSizeValue(totalUsed) + "]" +
            ", which is larger than the limit of [" +
            parentLimit + "/" + new ByteSizeValue(parentLimit) + "]");
        message.append(", usages [");
        message.append(this.breakers.entrySet().stream().map(e -> {
            final CircuitBreaker breaker = e.getValue();
            final long breakerUsed = (long)(breaker.getUsed() * breaker.getOverhead());
            return e.getKey() + "=" + breakerUsed + "/" + new ByteSizeValue(breakerUsed);
        }).collect(Collectors.joining(", ")));
        message.append("]");
        throw new CircuitBreakingException(message.toString(), totalUsed, parentLimit);
    }
}
 
Example #21
Source File: MockBigArrays.java    From crate with Apache License 2.0 5 votes vote down vote up
private MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerService, boolean checkBreaker) {
    super(recycler, breakerService, CircuitBreaker.REQUEST, checkBreaker);
    this.recycler = recycler;
    this.breakerService = breakerService;
    long seed;
    try {
        seed = SeedUtils.parseSeed(RandomizedContext.current().getRunnerSeedAsString());
    } catch (IllegalStateException e) { // rest tests don't run randomized and have no context
        seed = 0;
    }
    random = new Random(seed);
}
 
Example #22
Source File: NestedLoopOperation.java    From crate with Apache License 2.0 5 votes vote down vote up
public NestedLoopOperation(int numLeftCols,
                           int numRightCols,
                           RowConsumer nlResultConsumer,
                           Predicate<Row> joinPredicate,
                           JoinType joinType,
                           CircuitBreaker circuitBreaker,
                           RamAccounting ramAccounting,
                           List<DataType<?>> leftSideColumnTypes,
                           long estimatedRowsSizeLeft,
                           long estimatedNumberOfRowsLeft,
                           boolean blockNestedLoop) {
    this.resultConsumer = nlResultConsumer;
    this.leftConsumer = new CapturingRowConsumer(false, nlResultConsumer.completionFuture());
    this.rightConsumer = new CapturingRowConsumer(true, nlResultConsumer.completionFuture());
    CompletableFuture.allOf(leftConsumer.capturedBatchIterator(), rightConsumer.capturedBatchIterator())
        .whenComplete((result, failure) -> {
            if (failure == null) {
                BatchIterator<Row> nlIterator = createNestedLoopIterator(
                    leftConsumer.capturedBatchIterator().join(),
                    numLeftCols,
                    rightConsumer.capturedBatchIterator().join(),
                    numRightCols,
                    joinType,
                    joinPredicate,
                    circuitBreaker,
                    ramAccounting,
                    leftSideColumnTypes,
                    estimatedRowsSizeLeft,
                    estimatedNumberOfRowsLeft,
                    blockNestedLoop
                );
                nlResultConsumer.accept(nlIterator, null);
            } else {
                nlResultConsumer.accept(null, failure);
            }
        });
}
 
Example #23
Source File: RamBlockSizeCalculator.java    From crate with Apache License 2.0 5 votes vote down vote up
public RamBlockSizeCalculator(int defaultBlockSize,
                              CircuitBreaker circuitBreaker,
                              long estimatedRowSizeForLeft,
                              long numberOfRowsForLeft) {
    this.defaultBlockSize = defaultBlockSize;
    this.circuitBreaker = circuitBreaker;
    this.estimatedRowSizeForLeft = estimatedRowSizeForLeft;
    this.numberOfRowsForLeft = numberOfRowsForLeft;
}
 
Example #24
Source File: NodeFetchOperation.java    From crate with Apache License 2.0 5 votes vote down vote up
public NodeFetchOperation(ThreadPoolExecutor executor,
                          int numProcessors,
                          JobsLogs jobsLogs,
                          TasksService tasksService,
                          CircuitBreaker circuitBreaker) {
    this.executor = executor;
    this.numProcessors = numProcessors;
    this.jobsLogs = jobsLogs;
    this.tasksService = tasksService;
    this.circuitBreaker = circuitBreaker;
}
 
Example #25
Source File: SqlHttpHandler.java    From crate with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<XContentBuilder> executeSimpleRequest(Session session,
                                                                String stmt,
                                                                List<Object> args,
                                                                boolean includeTypes) throws IOException {
    long startTimeInNs = System.nanoTime();
    session.parse(UNNAMED, stmt, emptyList());
    session.bind(UNNAMED, UNNAMED, args == null ? emptyList() : args, null);
    DescribeResult description = session.describe('P', UNNAMED);
    List<Symbol> resultFields = description.getFields();
    ResultReceiver<XContentBuilder> resultReceiver;
    if (resultFields == null) {
        resultReceiver = new RestRowCountReceiver(JsonXContent.contentBuilder(), startTimeInNs, includeTypes);
    } else {
        CircuitBreaker breaker = circuitBreakerProvider.apply(HierarchyCircuitBreakerService.QUERY);
        RamAccounting ramAccounting = new BlockBasedRamAccounting(
            b -> breaker.addEstimateBytesAndMaybeBreak(b, "http-result"),
            MAX_BLOCK_SIZE_IN_BYTES);
        resultReceiver = new RestResultSetReceiver(
            JsonXContent.contentBuilder(),
            resultFields,
            startTimeInNs,
            new RowAccountingWithEstimators(
                Symbols.typeView(resultFields),
                ramAccounting
            ),
            includeTypes
        );
        resultReceiver.completionFuture().whenComplete((result, error) -> ramAccounting.close());
    }
    session.execute(UNNAMED, 0, resultReceiver);
    return session.sync()
        .thenCompose(ignored -> resultReceiver.completionFuture());
}
 
Example #26
Source File: RamAccountingQueue.java    From crate with Apache License 2.0 5 votes vote down vote up
public RamAccountingQueue(Queue<T> delegate, CircuitBreaker breaker, SizeEstimator<T> sizeEstimator) {
    this.delegate = delegate;
    this.breaker = breaker;
    this.sizeEstimator = sizeEstimator;
    // create a non-breaking (thread-safe) instance as this component will check the breaker limits by itself
    this.ramAccounting = new ConcurrentRamAccounting(
        breaker::addWithoutBreaking,
        bytes -> breaker.addWithoutBreaking(- bytes)
    );
    this.exceeded = new AtomicBoolean(false);
}
 
Example #27
Source File: HierarchyCircuitBreakerService.java    From crate with Apache License 2.0 5 votes vote down vote up
private void setRequestBreakerLimit(ByteSizeValue newRequestMax, Double newRequestOverhead) {
    BreakerSettings newRequestSettings = new BreakerSettings(CircuitBreaker.REQUEST, newRequestMax.getBytes(), newRequestOverhead,
            HierarchyCircuitBreakerService.this.requestSettings.getType());
    registerBreaker(newRequestSettings);
    HierarchyCircuitBreakerService.this.requestSettings = newRequestSettings;
    LOGGER.info("Updated breaker settings request: {}", newRequestSettings);
}
 
Example #28
Source File: HierarchyCircuitBreakerService.java    From crate with Apache License 2.0 5 votes vote down vote up
private void setFieldDataBreakerLimit(ByteSizeValue newFielddataMax, Double newFielddataOverhead) {
    long newFielddataLimitBytes = newFielddataMax == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getLimit() : newFielddataMax.getBytes();
    newFielddataOverhead = newFielddataOverhead == null ? HierarchyCircuitBreakerService.this.fielddataSettings.getOverhead() : newFielddataOverhead;
    BreakerSettings newFielddataSettings = new BreakerSettings(CircuitBreaker.FIELDDATA, newFielddataLimitBytes, newFielddataOverhead,
            HierarchyCircuitBreakerService.this.fielddataSettings.getType());
    registerBreaker(newFielddataSettings);
    HierarchyCircuitBreakerService.this.fielddataSettings = newFielddataSettings;
    LOGGER.info("Updated breaker settings field data: {}", newFielddataSettings);
}
 
Example #29
Source File: HierarchyCircuitBreakerService.java    From crate with Apache License 2.0 5 votes vote down vote up
private void setAccountingBreakerLimit(ByteSizeValue newAccountingMax, Double newAccountingOverhead) {
    BreakerSettings newAccountingSettings = new BreakerSettings(CircuitBreaker.ACCOUNTING, newAccountingMax.getBytes(),
        newAccountingOverhead, HierarchyCircuitBreakerService.this.inFlightRequestsSettings.getType());
    registerBreaker(newAccountingSettings);
    HierarchyCircuitBreakerService.this.accountingSettings = newAccountingSettings;
    LOGGER.info("Updated breaker settings for accounting requests: {}", newAccountingSettings);
}
 
Example #30
Source File: BigArrays.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Adjust the circuit breaker with the given delta, if the delta is
 * negative, or checkBreaker is false, the breaker will be adjusted
 * without tripping.  If the data was already created before calling
 * this method, and the breaker trips, we add the delta without breaking
 * to account for the created data.  If the data has not been created yet,
 * we do not add the delta to the breaker if it trips.
 */
void adjustBreaker(final long delta, final boolean isDataAlreadyCreated) {
    if (this.breakerService != null) {
        CircuitBreaker breaker = this.breakerService.getBreaker(breakerName);
        if (this.checkBreaker) {
            // checking breaker means potentially tripping, but it doesn't
            // have to if the delta is negative
            if (delta > 0) {
                try {
                    breaker.addEstimateBytesAndMaybeBreak(delta, "<reused_arrays>");
                } catch (CircuitBreakingException e) {
                    if (isDataAlreadyCreated) {
                        // since we've already created the data, we need to
                        // add it so closing the stream re-adjusts properly
                        breaker.addWithoutBreaking(delta);
                    }
                    // re-throw the original exception
                    throw e;
                }
            } else {
                breaker.addWithoutBreaking(delta);
            }
        } else {
            // even if we are not checking the breaker, we need to adjust
            // its' totals, so add without breaking
            breaker.addWithoutBreaking(delta);
        }
    }
}