Java Code Examples for org.apache.calcite.rel.RelNode#estimateRowCount()

protected boolean create2PhasePlan(RelOptRuleCall call, DrillAggregateRel aggregate) {
  PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
  RelNode child = call.rel(0).getInputs().get(0);
  boolean smallInput =
      child.estimateRowCount(child.getCluster().getMetadataQuery()) < settings.getSliceTarget();
  if (!settings.isMultiPhaseAggEnabled() || settings.isSingleMode()
      // Can override a small child - e.g., for testing with a small table
      || (smallInput && !settings.isForce2phaseAggr())) {
    return false;
  }

  for (AggregateCall aggCall : aggregate.getAggCallList()) {
    String name = aggCall.getAggregation().getName();
    if (!(name.equals(SqlKind.SUM.name())
        || name.equals(SqlKind.MIN.name())
        || name.equals(SqlKind.MAX.name())
        || name.equals(SqlKind.COUNT.name())
        || name.equals("$SUM0"))) {
      return false;
    }
  }
  return true;
}
 

@Override
public Double getDistinctRowCount(RelNode rel, RelMetadataQuery mq, ImmutableBitSet groupKey, RexNode predicate) {
  if (rel instanceof DrillScanRelBase) {                  // Applies to both Drill Logical and Physical Rels
    if (!DrillRelOptUtil.guessRows(rel)) {
      DrillTable table = Utilities.getDrillTable(rel.getTable());
      return getDistinctRowCountInternal(((DrillScanRelBase) rel), mq, table, groupKey, rel.getRowType(), predicate);
    } else {
      /* If we are not using statistics OR there is no table or metadata (stats) table associated with scan,
       * estimate the distinct row count. Consistent with the estimation of Aggregate row count in
       * RelMdRowCount: distinctRowCount = rowCount * 10%.
       */
      if (rel instanceof DrillScanRel) {
        // The existing Drill behavior is to only use this estimation for DrillScanRel and not ScanPrel.
        // TODO: We may potentially do it for ScanPrel (outside the scope of statistics)
        return rel.estimateRowCount(mq) * 0.1;
      }
    }
  } else if (rel instanceof SingleRel && !DrillRelOptUtil.guessRows(rel)) {
    if (rel instanceof Window) {
      int childFieldCount = ((Window) rel).getInput().getRowType().getFieldCount();
      // For window aggregates delegate ndv to parent
      for (int bit : groupKey) {
        if (bit >= childFieldCount) {
          return super.getDistinctRowCount(rel, mq, groupKey, predicate);
        }
      }
    }
    return mq.getDistinctRowCount(((SingleRel) rel).getInput(), groupKey, predicate);
  } else if (rel instanceof DrillJoinRelBase && !DrillRelOptUtil.guessRows(rel)) {
    //Assume ndv is unaffected by the join
    return getDistinctRowCountInternal(((DrillJoinRelBase) rel), mq, groupKey, predicate);
  } else if (rel instanceof RelSubset && !DrillRelOptUtil.guessRows(rel)) {
    if (((RelSubset) rel).getBest() != null) {
      return mq.getDistinctRowCount(((RelSubset) rel).getBest(), groupKey, predicate);
    } else if (((RelSubset) rel).getOriginal() != null) {
      return mq.getDistinctRowCount(((RelSubset) rel).getOriginal(), groupKey, predicate);
    }
  }
  return super.getDistinctRowCount(rel, mq, groupKey, predicate);
}
 

/**
 * Generate a possible RuntimeFilter of a HashJoinPrel, left some BF parameters of the generated RuntimeFilter
 * to be set later.
 *
 * @param hashJoinPrel
 * @return null or a partial information RuntimeFilterDef
 */
private RuntimeFilterDef generateRuntimeFilter(HashJoinPrel hashJoinPrel) {
  JoinRelType joinRelType = hashJoinPrel.getJoinType();
  JoinInfo joinInfo = hashJoinPrel.analyzeCondition();
  boolean allowJoin = (joinInfo.isEqui()) && (joinRelType == JoinRelType.INNER || joinRelType == JoinRelType.RIGHT);
  if (!allowJoin) {
    return null;
  }
  //TODO check whether to enable RuntimeFilter according to the NDV percent
  /**
   double threshold = 0.5;
   double percent = leftNDV / rightDNV;
   if (percent > threshold ) {
   return null;
   }
   */

  List<BloomFilterDef> bloomFilterDefs = new ArrayList<>();
  //find the possible left scan node of the left join key
  ScanPrel probeSideScanPrel = null;
  RelNode left = hashJoinPrel.getLeft();
  RelNode right = hashJoinPrel.getRight();
  ExchangePrel exchangePrel = findRightExchangePrel(right);
  if (exchangePrel == null) {
    //Does not support the single fragment mode ,that is the right build side
    //can only be BroadcastExchangePrel or HashToRandomExchangePrel
    return null;
  }
  List<String> leftFields = left.getRowType().getFieldNames();
  List<String> rightFields = right.getRowType().getFieldNames();
  List<Integer> leftKeys = hashJoinPrel.getLeftKeys();
  List<Integer> rightKeys = hashJoinPrel.getRightKeys();
  RelMetadataQuery metadataQuery = left.getCluster().getMetadataQuery();
  int i = 0;
  for (Integer leftKey : leftKeys) {
    String leftFieldName = leftFields.get(leftKey);
    Integer rightKey = rightKeys.get(i++);
    String rightFieldName = rightFields.get(rightKey);

    //This also avoids the left field of the join condition with a function call.
    ScanPrel scanPrel = findLeftScanPrel(leftFieldName, left);
    if (scanPrel != null) {
      boolean encounteredBlockNode = containBlockNode((Prel) left, scanPrel);
      if (encounteredBlockNode) {
        continue;
      }
      //Collect NDV from the Metadata
      RelDataType scanRowType = scanPrel.getRowType();
      RelDataTypeField field = scanRowType.getField(leftFieldName, true, true);
      int index = field.getIndex();
      Double ndv = metadataQuery.getDistinctRowCount(scanPrel, ImmutableBitSet.of(index), null);
      if (ndv == null) {
        //If NDV is not supplied, we use the row count to estimate the ndv.
        ndv = left.estimateRowCount(metadataQuery) * 0.1;
      }
      int bloomFilterSizeInBytes = BloomFilter.optimalNumOfBytes(ndv.longValue(), fpp);
      bloomFilterSizeInBytes = bloomFilterSizeInBytes > bloomFilterMaxSizeInBytesDef ? bloomFilterMaxSizeInBytesDef : bloomFilterSizeInBytes;
      //left the local parameter to be set later.
      BloomFilterDef bloomFilterDef = new BloomFilterDef(bloomFilterSizeInBytes, false, leftFieldName, rightFieldName);
      bloomFilterDef.setLeftNDV(ndv);
      bloomFilterDefs.add(bloomFilterDef);
      toAddRuntimeFilter.add(scanPrel);
      probeSideScanPrel = scanPrel;
    }
  }
  if (bloomFilterDefs.size() > 0) {
    //left sendToForeman parameter to be set later.
    RuntimeFilterDef runtimeFilterDef = new RuntimeFilterDef(true, false, bloomFilterDefs, false, -1);
    probeSideScan2hj.put(probeSideScanPrel, hashJoinPrel);
    return runtimeFilterDef;
  }
  return null;
}
 

private Double getDistinctRowCountFromEstimateRowCount(RelNode rel, RelMetadataQuery mq, ImmutableBitSet groupKey, RexNode predicate) {
  final int groupKeySize = groupKey.cardinality();
  return rel.estimateRowCount(mq) * (1.0 - Math.pow(0.9, groupKeySize)) * RelMdUtil.guessSelectivity(predicate);
}
 

/** Catch-all implementation for
 * {@link BuiltInMetadata.RowCount#getRowCount()},
 * invoked using reflection.
 *
 * @see org.apache.calcite.rel.metadata.RelMetadataQuery#getRowCount(RelNode)
 */
public Double getRowCount(RelNode rel, RelMetadataQuery mq) {
  return rel.estimateRowCount(mq);
}
 

/** Catch-all implementation for
 * {@link BuiltInMetadata.RowCount#getRowCount()},
 * invoked using reflection.
 *
 * @see org.apache.calcite.rel.metadata.RelMetadataQuery#getRowCount(RelNode)
 */
public Double getRowCount(RelNode rel, RelMetadataQuery mq) {
  return rel.estimateRowCount(mq);
}