Java Code Examples for org.apache.calcite.util.ImmutableBitSet#of()

private static void addWindows(Multimap<WindowKey, RexOver> windowMap, RexOver over, final int inputFieldCount) {
    final RexWindow aggWindow = over.getWindow();

    // Look up or create a window.
    RelCollation orderKeys = getCollation(Lists.newArrayList(Util.filter(aggWindow.orderKeys, rexFieldCollation ->
    // If ORDER BY references constant (i.e. RexInputRef),
    // then we can ignore such ORDER BY key.
    rexFieldCollation.left instanceof RexLocalRef)));
    ImmutableBitSet groupSet = ImmutableBitSet.of(getProjectOrdinals(aggWindow.partitionKeys));
    final int groupLength = groupSet.length();
    if (inputFieldCount < groupLength) {
        // If PARTITION BY references constant, we can ignore such partition key.
        // All the inputs after inputFieldCount are literals, thus we can clear.
        groupSet = groupSet.except(ImmutableBitSet.range(inputFieldCount, groupLength));
    }

    WindowKey windowKey = new WindowKey(groupSet, orderKeys, aggWindow.isRows(), aggWindow.getLowerBound(),
            aggWindow.getUpperBound());
    windowMap.put(windowKey, over);
}
 

/** Tests a singleton grouping set {2}, whose power set has only two elements,
 * { {2}, {} }. */
@Test void testInduceGroupingType1() {
  final ImmutableBitSet groupSet = ImmutableBitSet.of(2);

  // Could be ROLLUP but we prefer CUBE
  List<ImmutableBitSet> groupSets = new ArrayList<>();
  groupSets.add(groupSet);
  groupSets.add(ImmutableBitSet.of());
  assertEquals(Aggregate.Group.CUBE,
      Aggregate.Group.induce(groupSet, groupSets));

  groupSets = new ArrayList<>();
  groupSets.add(ImmutableBitSet.of());
  assertEquals(Aggregate.Group.OTHER,
      Aggregate.Group.induce(groupSet, groupSets));

  groupSets = new ArrayList<>();
  groupSets.add(groupSet);
  assertEquals(Aggregate.Group.SIMPLE,
      Aggregate.Group.induce(groupSet, groupSets));

  groupSets = new ArrayList<>();
  assertEquals(Aggregate.Group.OTHER,
      Aggregate.Group.induce(groupSet, groupSets));
}
 

@Override
public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
  List<Integer> groupingCols = Lists.newArrayList();
  groupingCols.add(0);
  for (int groupingCol : groupSet.asList()) {
    groupingCols.add(groupingCol + 1);
  }

  ImmutableBitSet groupingSet = ImmutableBitSet.of(groupingCols);
  List<ImmutableBitSet> groupingSets = Lists.newArrayList();
  groupingSets.add(groupingSet);
  List<AggregateCall> aggregateCalls = Lists.newArrayList();
  for (AggregateCall aggCall : aggCalls) {
    List<Integer> arglist = Lists.newArrayList();
    for (int arg : aggCall.getArgList()) {
      arglist.add(arg + 1);
    }
    aggregateCalls.add(AggregateCall.create(aggCall.getAggregation(), aggCall.isDistinct(),
            aggCall.isApproximate(), arglist, aggCall.filterArg, aggCall.type, aggCall.name));
  }
  return (Prel) copy(traitSet, children.get(0),indicator,groupingSet,groupingSets, aggregateCalls);
}
 

public Double visitInputRef(RexInputRef var) {
  int index = var.getIndex();
  ImmutableBitSet col = ImmutableBitSet.of(index);
  Double distinctRowCount =
      mq.getDistinctRowCount(rel.getInput(), col, null);
  if (distinctRowCount == null) {
    return null;
  } else {
    return numDistinctVals(distinctRowCount, mq.getRowCount(rel));
  }
}
 

private static void addWindows(
    Multimap<WindowKey, RexOver> windowMap,
    RexOver over, final int inputFieldCount) {
  final RexWindow aggWindow = over.getWindow();

  // Look up or create a window.
  RelCollation orderKeys = getCollation(
      Lists.newArrayList(
          Util.filter(aggWindow.orderKeys,
              rexFieldCollation ->
                  // If ORDER BY references constant (i.e. RexInputRef),
                  // then we can ignore such ORDER BY key.
                  rexFieldCollation.left instanceof RexLocalRef)));
  ImmutableBitSet groupSet =
      ImmutableBitSet.of(getProjectOrdinals(aggWindow.partitionKeys));
  final int groupLength = groupSet.length();
  if (inputFieldCount < groupLength) {
    // If PARTITION BY references constant, we can ignore such partition key.
    // All the inputs after inputFieldCount are literals, thus we can clear.
    groupSet =
        groupSet.except(ImmutableBitSet.range(inputFieldCount, groupLength));
  }

  WindowKey windowKey =
      new WindowKey(
          groupSet, orderKeys, aggWindow.isRows(),
          aggWindow.getLowerBound(), aggWindow.getUpperBound());
  windowMap.put(windowKey, over);
}
 

private void validateCubeLatticeFilter(Lattice.Builder latticeBuilder) {
  if (latticeBuilder.filter != null) {
    ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(latticeBuilder.filter);
    Set<Integer> dimSet = new HashSet<>();
    for (Dimension dimension : dimensions) {
      dimSet.add(latticeBuilder.resolveColumn(dimension.qualifiedCol).ordinal);
    }
    ImmutableBitSet dims = ImmutableBitSet.of(dimSet);
    if (!dims.contains(rCols)) {
      throw new RuntimeException("Cube filter is only allowed on dimensions");
    }
  }
}
 

@Test void testDistinctRowCountTable() {
  // no unique key information is available so return null
  RelNode rel = convertSql("select * from emp where deptno = 10");
  final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
  ImmutableBitSet groupKey =
      ImmutableBitSet.of(rel.getRowType().getFieldNames().indexOf("DEPTNO"));
  Double result = mq.getDistinctRowCount(rel, groupKey, null);
  assertThat(result, nullValue());
}
 

/**
 * Returns group-by keys with the remapped arguments for specified aggregate.
 *
 * @param groupSet ImmutableBitSet of aggregate rel node, whose group-by keys should be remapped.
 * @return {@link ImmutableBitSet} instance with remapped keys.
 */
public static ImmutableBitSet remapGroupSet(ImmutableBitSet groupSet) {
  List<Integer> newGroupSet = Lists.newArrayList();
  int groupSetToAdd = 0;
  for (int ignored : groupSet) {
    newGroupSet.add(groupSetToAdd++);
  }
  return ImmutableBitSet.of(newGroupSet);
}
 

/**
 * Gets the bit-set to the column ordinals in the source for columns that
 * intersect in the target.
 *
 * @param sourceRowType The source upon which to ordinate the bit set.
 * @param indexToField  The map of ordinals to target fields.
 */
public static ImmutableBitSet getOrdinalBitSet(
    RelDataType sourceRowType,
    Map<Integer, RelDataTypeField> indexToField) {
  ImmutableBitSet source = ImmutableBitSet.of(
      Lists.transform(sourceRowType.getFieldList(),
          RelDataTypeField::getIndex));
  ImmutableBitSet target =
      ImmutableBitSet.of(indexToField.keySet());
  return source.intersect(target);
}
 

public Double visitInputRef(RexInputRef var) {
  int index = var.getIndex();
  ImmutableBitSet col = ImmutableBitSet.of(index);
  Double distinctRowCount =
      mq.getDistinctRowCount(rel.getInput(), col, null);
  if (distinctRowCount == null) {
    return null;
  } else {
    return numDistinctVals(distinctRowCount, mq.getRowCount(rel));
  }
}
 

private static ImmutableBitSet bitSetOf(int... bits) {
  return ImmutableBitSet.of(bits);
}
 

public ImmutableBitSet rightSet() {
  return ImmutableBitSet.of(rightKeys);
}
 

public ImmutableBitSet leftSet() {
  return ImmutableBitSet.of(leftKeys);
}
 

/**
 * Determines whether a join of the dimension table in a semijoin can be
 * removed. It can be if the dimension keys are unique and the only fields
 * referenced from the dimension table are its semijoin keys. The semijoin
 * keys can be mapped to the corresponding keys from the fact table (because
 * of the equality condition associated with the semijoin keys). Therefore,
 * that's why the dimension table can be removed even though those fields
 * are referenced elsewhere in the query tree.
 *
 * @param multiJoin join factors being optimized
 * @param semiJoin semijoin under consideration
 * @param factIdx id of the fact table in the semijoin
 * @param dimIdx id of the dimension table in the semijoin
 */
private void removeJoin(
    LoptMultiJoin multiJoin,
    SemiJoin semiJoin,
    int factIdx,
    int dimIdx) {
  // if the dimension can be removed because of another semijoin, then
  // no need to proceed any further
  if (multiJoin.getJoinRemovalFactor(dimIdx) != null) {
    return;
  }

  // Check if the semijoin keys corresponding to the dimension table
  // are unique.  The semijoin will filter out the nulls.
  final ImmutableBitSet dimKeys = ImmutableBitSet.of(semiJoin.getRightKeys());
  final RelNode dimRel = multiJoin.getJoinFactor(dimIdx);
  if (!RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq, dimRel,
      dimKeys)) {
    return;
  }

  // check that the only fields referenced from the dimension table
  // in either its projection or join conditions are the dimension
  // keys
  ImmutableBitSet dimProjRefs = multiJoin.getProjFields(dimIdx);
  if (dimProjRefs == null) {
    int nDimFields = multiJoin.getNumFieldsInJoinFactor(dimIdx);
    dimProjRefs = ImmutableBitSet.range(0, nDimFields);
  }
  if (!dimKeys.contains(dimProjRefs)) {
    return;
  }
  int [] dimJoinRefCounts = multiJoin.getJoinFieldRefCounts(dimIdx);
  for (int i = 0; i < dimJoinRefCounts.length; i++) {
    if (dimJoinRefCounts[i] > 0) {
      if (!dimKeys.get(i)) {
        return;
      }
    }
  }

  // criteria met; keep track of the fact table and the semijoin that
  // allow the join of this dimension table to be removed
  multiJoin.setJoinRemovalFactor(dimIdx, factIdx);
  multiJoin.setJoinRemovalSemiJoin(dimIdx, semiJoin);

  // if the dimension table doesn't reference anything in its projection
  // and the only fields referenced in its joins are the dimension keys
  // of this semijoin, then we can decrement the join reference counts
  // corresponding to the fact table's semijoin keys, since the
  // dimension table doesn't need to use those keys
  if (dimProjRefs.cardinality() != 0) {
    return;
  }
  for (int i = 0; i < dimJoinRefCounts.length; i++) {
    if (dimJoinRefCounts[i] > 1) {
      return;
    } else if (dimJoinRefCounts[i] == 1) {
      if (!dimKeys.get(i)) {
        return;
      }
    }
  }
  int [] factJoinRefCounts = multiJoin.getJoinFieldRefCounts(factIdx);
  for (Integer key : semiJoin.getLeftKeys()) {
    factJoinRefCounts[key]--;
  }
}
 

/**
 * Computes a score relevant to applying a set of semijoins on a fact table.
 * The higher the score, the better.
 *
 * @param factRel fact table being filtered
 * @param dimRel dimension table that participates in semijoin
 * @param semiJoin semijoin between fact and dimension tables
 *
 * @return computed score of applying the dimension table filters on the
 * fact table
 */
private double computeScore(
    RelNode factRel,
    RelNode dimRel,
    LogicalJoin semiJoin) {
  // Estimate savings as a result of applying semijoin filter on fact
  // table.  As a heuristic, the selectivity of the semijoin needs to
  // be less than half.  There may be instances where an even smaller
  // selectivity value is required because of the overhead of
  // index lookups on a very large fact table.  Half was chosen as
  // a middle ground based on testing that was done with a large
  // data set.
  final ImmutableBitSet dimCols = ImmutableBitSet.of(semiJoin.analyzeCondition().rightKeys);
  final double selectivity =
      RelMdUtil.computeSemiJoinSelectivity(mq, factRel, dimRel, semiJoin);
  if (selectivity > .5) {
    return 0;
  }

  final RelOptCost factCost = mq.getCumulativeCost(factRel);

  // if not enough information, return a low score
  if (factCost == null) {
    return 0;
  }
  double savings =
      (1.0 - Math.sqrt(selectivity))
          * Math.max(1.0, factCost.getRows());

  // Additional savings if the dimension columns are unique.  We can
  // ignore nulls since they will be filtered out by the semijoin.
  boolean uniq =
      RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq,
          dimRel, dimCols);
  if (uniq) {
    savings *= 2.0;
  }

  // compute the cost of doing an extra scan on the dimension table,
  // including the distinct sort on top of the scan; if the dimension
  // columns are already unique, no need to add on the dup removal cost
  final Double dimSortCost = mq.getRowCount(dimRel);
  final Double dupRemCost = uniq ? 0 : dimSortCost;
  final RelOptCost dimCost = mq.getCumulativeCost(dimRel);
  if ((dimSortCost == null)
      || (dupRemCost == null)
      || (dimCost == null)) {
    return 0;
  }

  double dimRows = dimCost.getRows();
  if (dimRows < 1.0) {
    dimRows = 1.0;
  }
  return savings / dimRows;
}
 

/**
 * Computes a score relevant to applying a set of semijoins on a fact table.
 * The higher the score, the better.
 *
 * @param factRel fact table being filtered
 * @param dimRel dimension table that participates in semijoin
 * @param semiJoin semijoin between fact and dimension tables
 *
 * @return computed score of applying the dimension table filters on the
 * fact table
 */
private double computeScore(
    RelNode factRel,
    RelNode dimRel,
    SemiJoin semiJoin) {
  // Estimate savings as a result of applying semijoin filter on fact
  // table.  As a heuristic, the selectivity of the semijoin needs to
  // be less than half.  There may be instances where an even smaller
  // selectivity value is required because of the overhead of
  // index lookups on a very large fact table.  Half was chosen as
  // a middle ground based on testing that was done with a large
  // data set.
  final ImmutableBitSet dimCols = ImmutableBitSet.of(semiJoin.getRightKeys());
  final double selectivity =
      RelMdUtil.computeSemiJoinSelectivity(mq, factRel, dimRel, semiJoin);
  if (selectivity > .5) {
    return 0;
  }

  final RelOptCost factCost = mq.getCumulativeCost(factRel);

  // if not enough information, return a low score
  if (factCost == null) {
    return 0;
  }
  double savings =
      (1.0 - Math.sqrt(selectivity))
          * Math.max(1.0, factCost.getRows());

  // Additional savings if the dimension columns are unique.  We can
  // ignore nulls since they will be filtered out by the semijoin.
  boolean uniq =
      RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq,
          dimRel, dimCols);
  if (uniq) {
    savings *= 2.0;
  }

  // compute the cost of doing an extra scan on the dimension table,
  // including the distinct sort on top of the scan; if the dimension
  // columns are already unique, no need to add on the dup removal cost
  final Double dimSortCost = mq.getRowCount(dimRel);
  final Double dupRemCost = uniq ? 0 : dimSortCost;
  final RelOptCost dimCost = mq.getCumulativeCost(dimRel);
  if ((dimSortCost == null)
      || (dupRemCost == null)
      || (dimCost == null)) {
    return 0;
  }

  double dimRows = dimCost.getRows();
  if (dimRows < 1.0) {
    dimRows = 1.0;
  }
  return savings / dimRows;
}
 

public ImmutableBitSet rightSet() {
  return ImmutableBitSet.of(rightKeys);
}
 

/**
 * Sets weighting for each combination of factors, depending on which join
 * filters reference which factors. Greater weight is given to equality
 * conditions. Also, sets bitmaps indicating which factors are referenced by
 * each factor within join filters that are comparisons.
 */
public void setFactorWeights() {
  factorWeights = new int[nJoinFactors][nJoinFactors];
  factorsRefByFactor = new ImmutableBitSet[nJoinFactors];
  for (int i = 0; i < nJoinFactors; i++) {
    factorsRefByFactor[i] = ImmutableBitSet.of();
  }

  for (RexNode joinFilter : allJoinFilters) {
    ImmutableBitSet factorRefs = factorsRefByJoinFilter.get(joinFilter);

    // don't give weights to non-comparison expressions
    if (!(joinFilter instanceof RexCall)) {
      continue;
    }
    if (!joinFilter.isA(SqlKind.COMPARISON)) {
      continue;
    }

    // OR the factors referenced in this join filter into the
    // bitmaps corresponding to each of the factors; however,
    // exclude the bit corresponding to the factor itself
    for (int factor : factorRefs) {
      factorsRefByFactor[factor] =
          factorsRefByFactor[factor]
              .rebuild()
              .addAll(factorRefs)
              .clear(factor)
              .build();
    }

    if (factorRefs.cardinality() == 2) {
      int leftFactor = factorRefs.nextSetBit(0);
      int rightFactor = factorRefs.nextSetBit(leftFactor + 1);

      final RexCall call = (RexCall) joinFilter;
      ImmutableBitSet leftFields = fieldBitmap(call.getOperands().get(0));
      ImmutableBitSet leftBitmap = factorBitmap(leftFields);

      // filter contains only two factor references, one on each
      // side of the operator
      int weight;
      if (leftBitmap.cardinality() == 1) {
        // give higher weight to equi-joins
        switch (joinFilter.getKind()) {
        case EQUALS:
          weight = 3;
          break;
        default:
          weight = 2;
        }
      } else {
        // cross product of two tables
        weight = 1;
      }
      setFactorWeight(weight, leftFactor, rightFactor);
    } else {
      // multiple factor references -- set a weight for each
      // combination of factors referenced within the filter
      final List<Integer> list  = ImmutableIntList.copyOf(factorRefs);
      for (int outer : list) {
        for (int inner : list) {
          if (outer != inner) {
            setFactorWeight(1, outer, inner);
          }
        }
      }
    }
  }
}
 

/**
 * Sets weighting for each combination of factors, depending on which join
 * filters reference which factors. Greater weight is given to equality
 * conditions. Also, sets bitmaps indicating which factors are referenced by
 * each factor within join filters that are comparisons.
 */
public void setFactorWeights() {
  factorWeights = new int[nJoinFactors][nJoinFactors];
  factorsRefByFactor = new ImmutableBitSet[nJoinFactors];
  for (int i = 0; i < nJoinFactors; i++) {
    factorsRefByFactor[i] = ImmutableBitSet.of();
  }

  for (RexNode joinFilter : allJoinFilters) {
    ImmutableBitSet factorRefs = factorsRefByJoinFilter.get(joinFilter);

    // don't give weights to non-comparison expressions
    if (!(joinFilter instanceof RexCall)) {
      continue;
    }
    if (!joinFilter.isA(SqlKind.COMPARISON)) {
      continue;
    }

    // OR the factors referenced in this join filter into the
    // bitmaps corresponding to each of the factors; however,
    // exclude the bit corresponding to the factor itself
    for (int factor : factorRefs) {
      factorsRefByFactor[factor] =
          factorsRefByFactor[factor]
              .rebuild()
              .addAll(factorRefs)
              .clear(factor)
              .build();
    }

    if (factorRefs.cardinality() == 2) {
      int leftFactor = factorRefs.nextSetBit(0);
      int rightFactor = factorRefs.nextSetBit(leftFactor + 1);

      final RexCall call = (RexCall) joinFilter;
      ImmutableBitSet leftFields = fieldBitmap(call.getOperands().get(0));
      ImmutableBitSet leftBitmap = factorBitmap(leftFields);

      // filter contains only two factor references, one on each
      // side of the operator
      int weight;
      if (leftBitmap.cardinality() == 1) {
        // give higher weight to equi-joins
        switch (joinFilter.getKind()) {
        case EQUALS:
          weight = 3;
          break;
        default:
          weight = 2;
        }
      } else {
        // cross product of two tables
        weight = 1;
      }
      setFactorWeight(weight, leftFactor, rightFactor);
    } else {
      // multiple factor references -- set a weight for each
      // combination of factors referenced within the filter
      final List<Integer> list  = ImmutableIntList.copyOf(factorRefs);
      for (int outer : list) {
        for (int inner : list) {
          if (outer != inner) {
            setFactorWeight(1, outer, inner);
          }
        }
      }
    }
  }
}
 

public ImmutableBitSet leftSet() {
  return ImmutableBitSet.of(leftKeys);
}