Java Code Examples for org.apache.calcite.util.ImmutableBitSet#length()

private static void addWindows(Multimap<WindowKey, RexOver> windowMap, RexOver over, final int inputFieldCount) {
    final RexWindow aggWindow = over.getWindow();

    // Look up or create a window.
    RelCollation orderKeys = getCollation(Lists.newArrayList(Util.filter(aggWindow.orderKeys, rexFieldCollation ->
    // If ORDER BY references constant (i.e. RexInputRef),
    // then we can ignore such ORDER BY key.
    rexFieldCollation.left instanceof RexLocalRef)));
    ImmutableBitSet groupSet = ImmutableBitSet.of(getProjectOrdinals(aggWindow.partitionKeys));
    final int groupLength = groupSet.length();
    if (inputFieldCount < groupLength) {
        // If PARTITION BY references constant, we can ignore such partition key.
        // All the inputs after inputFieldCount are literals, thus we can clear.
        groupSet = groupSet.except(ImmutableBitSet.range(inputFieldCount, groupLength));
    }

    WindowKey windowKey = new WindowKey(groupSet, orderKeys, aggWindow.isRows(), aggWindow.getLowerBound(),
            aggWindow.getUpperBound());
    windowMap.put(windowKey, over);
}
 

private GroupKey groupKey_(ImmutableBitSet groupSet, boolean indicator, ImmutableList<ImmutableBitSet> groupSets) {
    if (groupSet.length() > peek().getRowType().getFieldCount()) {
        throw new IllegalArgumentException("out of bounds: " + groupSet);
    }
    Objects.requireNonNull(groupSets);
    final ImmutableList<RexNode> nodes = fields(ImmutableIntList.of(groupSet.toArray()));
    final List<ImmutableList<RexNode>> nodeLists = Util.transform(groupSets,
            bitSet -> fields(ImmutableIntList.of(bitSet.toArray())));
    return groupKey_(nodes, indicator, nodeLists);
}
 

private static void addWindows(
    Multimap<WindowKey, RexOver> windowMap,
    RexOver over, final int inputFieldCount) {
  final RexWindow aggWindow = over.getWindow();

  // Look up or create a window.
  RelCollation orderKeys = getCollation(
      Lists.newArrayList(
          Util.filter(aggWindow.orderKeys,
              rexFieldCollation ->
                  // If ORDER BY references constant (i.e. RexInputRef),
                  // then we can ignore such ORDER BY key.
                  rexFieldCollation.left instanceof RexLocalRef)));
  ImmutableBitSet groupSet =
      ImmutableBitSet.of(getProjectOrdinals(aggWindow.partitionKeys));
  final int groupLength = groupSet.length();
  if (inputFieldCount < groupLength) {
    // If PARTITION BY references constant, we can ignore such partition key.
    // All the inputs after inputFieldCount are literals, thus we can clear.
    groupSet =
        groupSet.except(ImmutableBitSet.range(inputFieldCount, groupLength));
  }

  WindowKey windowKey =
      new WindowKey(
          groupSet, orderKeys, aggWindow.isRows(),
          aggWindow.getLowerBound(), aggWindow.getUpperBound());
  windowMap.put(windowKey, over);
}
 

private GroupKey groupKey_(ImmutableBitSet groupSet,
    @Nonnull ImmutableList<ImmutableBitSet> groupSets) {
  if (groupSet.length() > peek().getRowType().getFieldCount()) {
    throw new IllegalArgumentException("out of bounds: " + groupSet);
  }
  Objects.requireNonNull(groupSets);
  final ImmutableList<RexNode> nodes = fields(groupSet);
  return groupKey_(nodes, Util.transform(groupSets, bitSet -> fields(bitSet)));
}
 

/**
 * Estimates the number of rows which would be produced by a GROUP BY on the
 * set of columns indicated by groupKey.
 * column").
 */
private Double getDistinctRowCountInternal(DrillScanRelBase scan, RelMetadataQuery mq, DrillTable table,
    ImmutableBitSet groupKey, RelDataType type, RexNode predicate) {
  double selectivity, rowCount;
  /* If predicate is present, determine its selectivity to estimate filtered rows.
   * Thereafter, compute the number of distinct rows.
   */
  selectivity = mq.getSelectivity(scan, predicate);
  rowCount = mq.getRowCount(scan);

  if (groupKey.length() == 0) {
    return selectivity * rowCount;
  }

  /* If predicate is present, determine its selectivity to estimate filtered rows. Thereafter,
   * compute the number of distinct rows
   */
  selectivity = mq.getSelectivity(scan, predicate);
  TableMetadata tableMetadata;
  try {
    tableMetadata = table.getGroupScan().getTableMetadata();
  } catch (IOException e) {
    // Statistics cannot be obtained, use default behaviour
    return scan.estimateRowCount(mq) * 0.1;
  }
  double s = 1.0;

  for (int i = 0; i < groupKey.length(); i++) {
    final String colName = type.getFieldNames().get(i);
    // Skip NDV, if not available
    if (!groupKey.get(i)) {
      continue;
    }
    ColumnStatistics columnStatistics = tableMetadata != null ? tableMetadata.getColumnStatistics(SchemaPath.getSimplePath(colName)) : null;
    Double ndv = columnStatistics != null ? (Double) columnStatistics.getStatistic(ColumnStatisticsKind.NVD) : null;
    if (ndv == null) {
      continue;
    }
    s *= 1 - ndv / rowCount;
  }
  if (s > 0 && s < 1.0) {
    return (1 - s) * selectivity * rowCount;
  } else if (s == 1.0) {
    // Could not get any NDV estimate from stats - probably stats not present for GBY cols. So Guess!
    return scan.estimateRowCount(mq) * 0.1;
  } else {
    /* rowCount maybe less than NDV(different source), sanity check OR NDV not used at all */
    return selectivity * rowCount;
  }
}
 

private Double getDistinctRowCountInternal(DrillJoinRelBase joinRel, RelMetadataQuery mq, ImmutableBitSet groupKey,
     RexNode predicate) {
  if (DrillRelOptUtil.guessRows(joinRel)) {
    return super.getDistinctRowCount(joinRel, mq, groupKey, predicate);
  }
  // Assume NDV is unaffected by the join when groupKey comes from one side of the join
  // Alleviates NDV over-estimates
  ImmutableBitSet.Builder leftMask = ImmutableBitSet.builder();
  ImmutableBitSet.Builder rightMask = ImmutableBitSet.builder();
  JoinRelType joinType = joinRel.getJoinType();
  RelNode left = joinRel.getInputs().get(0);
  RelNode right = joinRel.getInputs().get(1);
  RelMdUtil.setLeftRightBitmaps(groupKey, leftMask, rightMask,
      left.getRowType().getFieldCount());
  RexNode leftPred = null;
  RexNode rightPred = null;

  // Identify predicates which can be pushed onto the left and right sides of the join
  if (predicate != null) {
    List<RexNode> leftFilters = new ArrayList<>();
    List<RexNode> rightFilters = new ArrayList<>();
    List<RexNode> joinFilters = new ArrayList<>();
    List<RexNode> predList = RelOptUtil.conjunctions(predicate);
    RelOptUtil.classifyFilters(joinRel, predList, joinType, joinType == JoinRelType.INNER,
        !joinType.generatesNullsOnLeft(), !joinType.generatesNullsOnRight(), joinFilters,
            leftFilters, rightFilters);
    RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
    leftPred = RexUtil.composeConjunction(rexBuilder, leftFilters, true);
    rightPred = RexUtil.composeConjunction(rexBuilder, rightFilters, true);
  }

  Double leftDistRowCount = null;
  Double rightDistRowCount = null;
  double distRowCount = 1;
  ImmutableBitSet lmb = leftMask.build();
  ImmutableBitSet rmb = rightMask.build();
  // Get NDV estimates for the left and right side predicates, if applicable
  if (lmb.length() > 0) {
    leftDistRowCount = mq.getDistinctRowCount(left, lmb, leftPred);
    if (leftDistRowCount != null) {
      distRowCount = leftDistRowCount;
    }
  }
  if (rmb.length() > 0) {
    rightDistRowCount = mq.getDistinctRowCount(right, rmb, rightPred);
    if (rightDistRowCount != null) {
      distRowCount = rightDistRowCount;
    }
  }
  // Use max of NDVs from both sides of the join, if applicable
  if (leftDistRowCount != null && rightDistRowCount != null) {
    distRowCount = Math.max(leftDistRowCount, rightDistRowCount);
  }
  return RelMdUtil.numDistinctVals(distRowCount, mq.getRowCount(joinRel));
}