org.apache.calcite.util.ImmutableBitSet Java Examples

public MongoAggregate(
    RelOptCluster cluster,
    RelTraitSet traitSet,
    RelNode input,
    ImmutableBitSet groupSet,
    List<ImmutableBitSet> groupSets,
    List<AggregateCall> aggCalls)
    throws InvalidRelException {
  super(cluster, traitSet, ImmutableList.of(), input, groupSet, groupSets, aggCalls);
  assert getConvention() == MongoRel.CONVENTION;
  assert getConvention() == input.getConvention();

  for (AggregateCall aggCall : aggCalls) {
    if (aggCall.isDistinct()) {
      throw new InvalidRelException(
          "distinct aggregation not supported");
    }
  }
  switch (getGroupType()) {
  case SIMPLE:
    break;
  default:
    throw new InvalidRelException("unsupported group type: "
        + getGroupType());
  }
}
 

public static EnumerableBatchNestedLoopJoin create(
    RelNode left,
    RelNode right,
    RexNode condition,
    ImmutableBitSet requiredColumns,
    Set<CorrelationId> variablesSet,
    JoinRelType joinType) {
  final RelOptCluster cluster = left.getCluster();
  final RelMetadataQuery mq = cluster.getMetadataQuery();
  final RelTraitSet traitSet =
      cluster.traitSetOf(EnumerableConvention.INSTANCE)
          .replaceIfs(RelCollationTraitDef.INSTANCE,
              () -> RelMdCollation.enumerableBatchNestedLoopJoin(mq, left, right, joinType));
  return new EnumerableBatchNestedLoopJoin(
      cluster,
      traitSet,
      left,
      right,
      condition,
      variablesSet,
      requiredColumns,
      joinType);
}
 

InputSpecialOpFinder(
    BitSet rexRefs,
    ImmutableBitSet leftFields,
    ImmutableBitSet rightFields,
    final ImmutableBitSet strongFields,
    ExprCondition preserveExprCondition,
    List<RexNode> preserveLeft,
    List<RexNode> preserveRight) {
  super(true);
  this.rexRefs = rexRefs;
  this.leftFields = leftFields;
  this.rightFields = rightFields;
  this.preserveExprCondition = preserveExprCondition;
  this.preserveLeft = preserveLeft;
  this.preserveRight = preserveRight;

  this.strongFields = strongFields;
  this.strong = Strong.of(strongFields);
}
 

@Test
public void testGetPartitionStatsWithUnknownColumnStats() throws Exception {
	TestPartitionableSourceFactory.createTemporaryTable(tEnv, "PartT", true);
	createPartitionStats("A", 1);
	createPartitionStats("A", 2);
	createPartitionColumnStats("A", 2);

	RelNode t1 = ((PlannerBase) ((TableEnvironmentImpl) tEnv).getPlanner()).optimize(
			TableTestUtil.toRelNode(tEnv.sqlQuery("select id, name from PartT where part1 = 'A'")));
	FlinkRelMetadataQuery mq = FlinkRelMetadataQuery.reuseOrCreate(t1.getCluster().getMetadataQuery());
	assertEquals(200.0, mq.getRowCount(t1), 0.0);

	// long type
	assertNull(mq.getDistinctRowCount(t1, ImmutableBitSet.of(0), null));
	assertNull(mq.getColumnNullCount(t1, 0));
	assertNull(mq.getColumnInterval(t1, 0));

	// string type
	assertNull(mq.getDistinctRowCount(t1, ImmutableBitSet.of(1), null));
	assertNull(mq.getColumnNullCount(t1, 1));
}
 

/**
 * Trims unused fields from a relational expression.
 *
 * <p>We presume that all fields of the relational expression are wanted by
 * its consumer, so only trim fields that are not used within the tree.
 *
 * @param root Root node of relational expression
 * @return Trimmed relational expression
 */
public RelNode trim(RelNode root) {
  final int fieldCount = root.getRowType().getFieldCount();
  final ImmutableBitSet fieldsUsed = ImmutableBitSet.range(fieldCount);
  final Set<RelDataTypeField> extraFields = Collections.emptySet();
  final TrimResult trimResult =
      dispatchTrimFields(root, fieldsUsed, extraFields);
  if (!trimResult.right.isIdentity()) {
    throw new IllegalArgumentException();
  }
  if (SqlToRelConverter.SQL2REL_LOGGER.isDebugEnabled()) {
    SqlToRelConverter.SQL2REL_LOGGER.debug(
        RelOptUtil.dumpPlan("Plan after trimming unused fields",
            trimResult.left, SqlExplainFormat.TEXT,
            SqlExplainLevel.EXPPLAN_ATTRIBUTES));
  }
  return trimResult.left;
}
 

public void rewriteRel(LogicalCorrelate rel) {
  ImmutableBitSet.Builder newPos = ImmutableBitSet.builder();
  for (int pos : rel.getRequiredColumns()) {
    RelDataType corrFieldType =
        rel.getLeft().getRowType().getFieldList().get(pos)
            .getType();
    if (corrFieldType.isStruct()) {
      throw Util.needToImplement("correlation on structured type");
    }
    newPos.set(getNewForOldInput(pos));
  }
  LogicalCorrelate newRel =
      LogicalCorrelate.create(getNewForOldRel(rel.getLeft()),
          getNewForOldRel(rel.getRight()),
          rel.getCorrelationId(),
          newPos.build(),
          rel.getJoinType());
  setNewForOldRel(rel, newRel);
}
 

private MutableMatch(RelDataType rowType, MutableRel input,
     RexNode pattern, boolean strictStart, boolean strictEnd,
     Map<String, RexNode> patternDefinitions, Map<String, RexNode> measures,
     RexNode after, Map<String, ? extends SortedSet<String>> subsets,
     boolean allRows, ImmutableBitSet partitionKeys, RelCollation orderKeys,
     RexNode interval) {
  super(MutableRelType.MATCH, rowType, input);
  this.pattern = pattern;
  this.strictStart = strictStart;
  this.strictEnd = strictEnd;
  this.patternDefinitions = patternDefinitions;
  this.measures = measures;
  this.after = after;
  this.subsets = subsets;
  this.allRows = allRows;
  this.partitionKeys = partitionKeys;
  this.orderKeys = orderKeys;
  this.interval = interval;
}
 

/**
 * Given an expression, it will create all equivalent expressions resulting
 * from replacing all possible combinations of references in the mapping by
 * the corresponding expressions.
 *
 * @param rexBuilder rexBuilder
 * @param expr expression
 * @param mapping mapping
 * @return set of resulting expressions equivalent to the input expression
 */
@Nullable protected static Set<RexNode> createAllPossibleExpressions(RexBuilder rexBuilder,
    RexNode expr, Map<RexInputRef, Set<RexNode>> mapping) {
  // Extract input fields referenced by expression
  final ImmutableBitSet predFieldsUsed = extractInputRefs(expr);

  if (predFieldsUsed.isEmpty()) {
    // The unique expression is the input expression
    return ImmutableSet.of(expr);
  }

  try {
    return createAllPossibleExpressions(rexBuilder, expr, predFieldsUsed, mapping,
        new HashMap<>());
  } catch (UnsupportedOperationException e) {
    // There may be a RexNode unsupported by RexCopier, just return null
    return null;
  }
}
 

public Double getDistinctRowCount(Filter rel, RelMetadataQuery mq,
    ImmutableBitSet groupKey, RexNode predicate) {
  if (predicate == null || predicate.isAlwaysTrue()) {
    if (groupKey.isEmpty()) {
      return 1D;
    }
  }
  // REVIEW zfong 4/18/06 - In the Broadbase code, duplicates are not
  // removed from the two filter lists.  However, the code below is
  // doing so.
  RexNode unionPreds =
      RelMdUtil.unionPreds(
          rel.getCluster().getRexBuilder(),
          predicate,
          rel.getCondition());

  return mq.getDistinctRowCount(rel.getInput(), groupKey, unionPreds);
}
 

/**
 * Returns the
 * {@link BuiltInMetadata.DistinctRowCount#getDistinctRowCount(ImmutableBitSet, RexNode)}
 * statistic.
 *
 * @param rel       the relational expression
 * @param groupKey  column mask representing group by columns
 * @param predicate pre-filtered predicates
 * @return distinct row count for groupKey, filtered by predicate, or null
 * if no reliable estimate can be determined
 */
public Double getDistinctRowCount(
    RelNode rel,
    ImmutableBitSet groupKey,
    RexNode predicate) {
  for (;;) {
    try {
      Double result =
          distinctRowCountHandler.getDistinctRowCount(rel, this, groupKey,
              predicate);
      return validateResult(result);
    } catch (JaninoRelMetadataProvider.NoHandler e) {
      distinctRowCountHandler =
          revise(e.relClass, BuiltInMetadata.DistinctRowCount.DEF);
    }
  }
}
 

/**
 * @param fetch limit to fetch
 * @param collationIndexes index of fields as listed in query row output
 * @param collationDirections direction of sort
 * @param numericCollationIndexes flag of to determine sort comparator
 * @param queryOutputFieldNames query output fields
 *
 * @return always an non null Json Limit object
 */
private JsonLimit computeSort(@Nullable Integer fetch, List<Integer> collationIndexes,
    List<Direction> collationDirections, ImmutableBitSet numericCollationIndexes,
    List<String> queryOutputFieldNames) {
  final List<JsonCollation> collations;
  if (collationIndexes != null) {
    assert collationDirections != null;
    ImmutableList.Builder<JsonCollation> colBuilder = ImmutableList.builder();
    for (Pair<Integer, Direction> p : Pair.zip(collationIndexes, collationDirections)) {
      final String dimensionOrder = numericCollationIndexes.get(p.left)
          ? "numeric"
          : "lexicographic";
      colBuilder.add(
          new JsonCollation(queryOutputFieldNames.get(p.left),
              p.right == Direction.DESCENDING ? "descending" : "ascending", dimensionOrder));
    }
    collations = colBuilder.build();
  } else {
    collations = null;
  }
  return new JsonLimit("default", fetch, collations);
}
 

public AggregateNode(Compiler compiler, Aggregate rel) {
  super(compiler, rel);
  this.dataContext = compiler.getDataContext();

  ImmutableBitSet union = ImmutableBitSet.of();

  if (rel.getGroupSets() != null) {
    for (ImmutableBitSet group : rel.getGroupSets()) {
      union = union.union(group);
      groups.add(new Grouping(group));
    }
  }

  this.unionGroups = union;
  this.outputRowLength = unionGroups.cardinality()
      + rel.getAggCallList().size();

  ImmutableList.Builder<AccumulatorFactory> builder = ImmutableList.builder();
  for (AggregateCall aggregateCall : rel.getAggCallList()) {
    builder.add(getAccumulator(aggregateCall, false));
  }
  accumulatorFactories = builder.build();
}
 

/** Gets all fields in an aggregate. */
public static Set<Integer> getAllFields2(ImmutableBitSet groupSet,
    List<AggregateCall> aggCallList) {
  final Set<Integer> allFields = new TreeSet<>();
  allFields.addAll(groupSet.asList());
  for (AggregateCall aggregateCall : aggCallList) {
    allFields.addAll(aggregateCall.getArgList());
    if (aggregateCall.filterArg >= 0) {
      allFields.add(aggregateCall.filterArg);
    }
    allFields.addAll(RelCollations.ordinals(aggregateCall.collation));
  }
  return allFields;
}
 

public Statistic getStatistic() {
  return new Statistic() {
    public Double getRowCount() {
      return table.rowCount;
    }

    public boolean isKey(ImmutableBitSet columns) {
      return table.isKey(columns);
    }

    public List<ImmutableBitSet> getKeys() {
      return table.getKeys();
    }

    public List<RelReferentialConstraint> getReferentialConstraints() {
      return table.getReferentialConstraints();
    }

    public List<RelCollation> getCollations() {
      return table.collationList;
    }

    public RelDistribution getDistribution() {
      return table.getDistribution();
    }
  };
}
 

private static List<Integer> remap(ImmutableBitSet groupSet,
    List<Integer> argList) {
  ImmutableIntList list = ImmutableIntList.of();
  for (int arg : argList) {
    list = list.append(remap(groupSet, arg));
  }
  return list;
}
 

@Deprecated // to be removed before 2.0
public BindableAggregate(RelOptCluster cluster, RelTraitSet traitSet,
    RelNode input, boolean indicator, ImmutableBitSet groupSet,
    List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls)
    throws InvalidRelException {
  this(cluster, traitSet, input, groupSet, groupSets, aggCalls);
  checkIndicator(indicator);
}
 

/** Creates a PigAggregate. */
public PigAggregate(RelOptCluster cluster, RelTraitSet traitSet,
    RelNode input, ImmutableBitSet groupSet,
    List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
  super(cluster, traitSet, ImmutableList.of(), input, groupSet, groupSets, aggCalls);
  assert getConvention() == PigRel.CONVENTION;
}
 

/**
 * Creates a MutableAggregate.
 *
 * @param input     Input relational expression
 * @param groupSet  Bit set of grouping fields
 * @param groupSets List of all grouping sets; null for just {@code groupSet}
 * @param aggCalls  Collection of calls to aggregate functions
 */
public static MutableAggregate of(MutableRel input, ImmutableBitSet groupSet,
    ImmutableList<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
  RelDataType rowType =
      Aggregate.deriveRowType(input.cluster.getTypeFactory(),
          input.rowType, false, groupSet, groupSets, aggCalls);
  return new MutableAggregate(input, rowType, groupSet,
      groupSets, aggCalls);
}
 

@Deprecated // to be removed before 2.0
public LogicalAggregate(
    RelOptCluster cluster,
    RelNode child,
    boolean indicator,
    ImmutableBitSet groupSet,
    List<ImmutableBitSet> groupSets,
    List<AggregateCall> aggCalls) {
  this(cluster, cluster.traitSetOf(Convention.NONE), child, indicator,
      groupSet, groupSets, aggCalls);
}
 

@Test void testColumnUniquenessForMinusWithConstantColumns() {
  final String sql = ""
      + "select deptno, sal\n"
      + "from (select distinct deptno, sal from emp)\n"
      + "where sal=1000\n"
      + "except all\n"
      + "select deptno, sal from emp\n";
  final RelNode rel = convertSql(sql);
  final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
  assertThat(rel.getRowType().getFieldNames().toString(),
      is("[DEPTNO, SAL]"));
  assertThat(mq.areColumnsUnique(rel, ImmutableBitSet.of(0)), is(true));
  assertThat(mq.areColumnsUnique(rel, ImmutableBitSet.of(0, 1)), is(true));
}
 

/**
 * Rewrites a scalar sub-query into an
 * {@link org.apache.calcite.rel.core.Aggregate}.
 *
 * @param e            IN sub-query to rewrite
 * @param variablesSet A set of variables used by a relational
 *                     expression of the specified RexSubQuery
 * @param builder      Builder
 * @param offset       Offset to shift {@link RexInputRef}
 *
 * @return Expression that may be used to replace the RexSubQuery
 */
private RexNode rewriteScalarQuery(RexSubQuery e, Set<CorrelationId> variablesSet, RelBuilder builder,
        int inputCount, int offset) {
    builder.push(e.getRel());
    final RelMetadataQuery mq = e.getRel().getCluster().getMetadataQuery();
    final Boolean unique = mq.areColumnsUnique(builder.peek(), ImmutableBitSet.of());
    if (unique == null || !unique) {
        builder.aggregate(builder.groupKey(),
                builder.aggregateCall(SqlStdOperatorTable.SINGLE_VALUE, builder.field(0)));
    }
    builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
    return field(builder, inputCount, offset);
}
 

/**
 * Returns the
 * {@link BuiltInMetadata.PopulationSize#getPopulationSize(ImmutableBitSet)}
 * statistic.
 *
 * @param rel      the relational expression
 * @param groupKey column mask representing the subset of columns for which
 *                 the row count will be determined
 * @return distinct row count for the given groupKey, or null if no reliable
 * estimate can be determined
 *
 */
public Double getPopulationSize(RelNode rel,
    ImmutableBitSet groupKey) {
  for (;;) {
    try {
      Double result =
          populationSizeHandler.getPopulationSize(rel, this, groupKey);
      return RelMdUtil.validateResult(result);
    } catch (JaninoRelMetadataProvider.NoHandler e) {
      populationSizeHandler =
          revise(e.relClass, BuiltInMetadata.PopulationSize.DEF);
    }
  }
}
 

/** Creates a ProfilerLatticeStatisticProvider. */
private ProfilerLatticeStatisticProvider(Lattice lattice) {
  Objects.requireNonNull(lattice);
  this.profile = Suppliers.memoize(() -> {
    final ProfilerImpl profiler =
        ProfilerImpl.builder()
            .withPassSize(200)
            .withMinimumSurprise(0.3D)
            .build();
    final List<Profiler.Column> columns = new ArrayList<>();
    for (Lattice.Column column : lattice.columns) {
      columns.add(new Profiler.Column(column.ordinal, column.alias));
    }
    final String sql =
        lattice.sql(ImmutableBitSet.range(lattice.columns.size()),
            false, ImmutableList.of());
    final Table table =
        new MaterializationService.DefaultTableFactory()
            .createTable(lattice.rootSchema, sql, ImmutableList.of());
    final ImmutableList<ImmutableBitSet> initialGroups =
        ImmutableList.of();
    final Enumerable<List<Comparable>> rows =
        ((ScannableTable) table).scan(null)
            .select(values -> {
              for (int i = 0; i < values.length; i++) {
                if (values[i] == null) {
                  values[i] = NullSentinel.INSTANCE;
                }
              }
              //noinspection unchecked
              return (List<Comparable>) (List) Arrays.asList(values);
            });
    return profiler.profile(rows, columns, initialGroups);
  })::get;
}
 

private static long groupValue(ImmutableBitSet fullGroupSet,
    ImmutableBitSet groupSet) {
  long v = 0;
  long x = 1L << (fullGroupSet.cardinality() - 1);
  assert fullGroupSet.contains(groupSet);
  for (int i : fullGroupSet) {
    if (!groupSet.get(i)) {
      v |= x;
    }
    x >>= 1;
  }
  return v;
}
 

@Test void testDistinctRowCountTable() {
  // no unique key information is available so return null
  RelNode rel = convertSql("select * from emp where deptno = 10");
  final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
  ImmutableBitSet groupKey =
      ImmutableBitSet.of(rel.getRowType().getFieldNames().indexOf("DEPTNO"));
  Double result = mq.getDistinctRowCount(rel, groupKey, null);
  assertThat(result, nullValue());
}
 

public static List<NamedExpression> aggsToExpr(
    RelDataType rowType, RelNode input, ImmutableBitSet groupSet, List<AggregateCall> aggCalls) {
  final List<String> fields = rowType.getFieldNames();
  final List<String> childFields = input.getRowType().getFieldNames();
  final List<NamedExpression> aggExprs = Lists.newArrayList();
  for (Ord<AggregateCall> aggCall : Ord.zip(aggCalls)) {
    int aggExprOrdinal = groupSet.cardinality() + aggCall.i;
    FieldReference ref = FieldReference.getWithQuotedRef(fields.get(aggExprOrdinal));
    LogicalExpression expr = toExpr(aggCall.e, childFields);
    NamedExpression ne = new NamedExpression(expr, ref);
    aggExprs.add(ne);
  }
  return aggExprs;
}
 

public Double visitInputRef(RexInputRef var) {
  int index = var.getIndex();
  ImmutableBitSet col = ImmutableBitSet.of(index);
  Double distinctRowCount =
      mq.getDistinctRowCount(rel.getInput(), col, null);
  if (distinctRowCount == null) {
    return null;
  } else {
    return numDistinctVals(distinctRowCount, mq.getRowCount(rel));
  }
}
 

public static List<RexNode> identityProjects(RelDataType type, ImmutableBitSet selectedColumns) {
  List<RexNode> projects = new ArrayList<>();
  if (selectedColumns == null) {
    selectedColumns = ImmutableBitSet.range(type.getFieldCount());
  }
  for (Pair<Integer,RelDataTypeField> pair : Pair.zip(selectedColumns, type.getFieldList())) {
    projects.add(new RexInputRef(pair.left, pair.right.getType()));
  }
  return projects;
}
 

private static ImmutableList<ImmutableBitSet> remap(ImmutableBitSet groupSet,
    Iterable<ImmutableBitSet> bitSets) {
  final ImmutableList.Builder<ImmutableBitSet> builder =
      ImmutableList.builder();
  for (ImmutableBitSet bitSet : bitSets) {
    builder.add(remap(groupSet, bitSet));
  }
  return builder.build();
}
 

private GroupKey groupKey_(ImmutableBitSet groupSet,
    @Nonnull ImmutableList<ImmutableBitSet> groupSets) {
  if (groupSet.length() > peek().getRowType().getFieldCount()) {
    throw new IllegalArgumentException("out of bounds: " + groupSet);
  }
  Objects.requireNonNull(groupSets);
  final ImmutableList<RexNode> nodes = fields(groupSet);
  return groupKey_(nodes, Util.transform(groupSets, bitSet -> fields(bitSet)));
}