org.apache.calcite.tools.RuleSets Java Examples

@Test void testUseMergeJoin() {
  final String sql = "select /*+ use_merge_join(emp, dept) */\n"
      + "ename, job, sal, dept.name\n"
      + "from emp join dept on emp.deptno = dept.deptno";
  RelOptPlanner planner = new VolcanoPlanner();
  planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
  planner.addRelTraitDef(RelCollationTraitDef.INSTANCE);
  Tester tester1 = tester.withDecorrelation(true)
      .withClusterFactory(
          relOptCluster -> RelOptCluster.create(planner, relOptCluster.getRexBuilder()));
  final RelNode rel = tester1.convertSqlToRel(sql).rel;
  RuleSet ruleSet = RuleSets.ofList(
      EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE,
      EnumerableRules.ENUMERABLE_JOIN_RULE,
      EnumerableRules.ENUMERABLE_PROJECT_RULE,
      EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE,
      EnumerableRules.ENUMERABLE_SORT_RULE,
      AbstractConverter.ExpandConversionRule.INSTANCE);
  Program program = Programs.of(ruleSet);
  RelTraitSet toTraits = rel
      .getCluster()
      .traitSet()
      .replace(EnumerableConvention.INSTANCE);

  RelNode relAfter = program.run(planner, rel, toTraits,
      Collections.emptyList(), Collections.emptyList());

  String planAfter = NL + RelOptUtil.toString(relAfter);
  getDiffRepos().assertEquals("planAfter", "${planAfter}", planAfter);
}
 

public static RelNode toPhysical(RelNode rel, Consumer<RelOptPlanner> setting) {
    final RelOptPlanner planner = rel.getCluster().getPlanner();
    planner.clear();
    setting.accept(planner);
    planner.addRule(new RelOptRule(operand(MycatTransientSQLTableScan.class, none()), RelFactories.LOGICAL_BUILDER, "MycatTransientSQLTableScan") {

        @Override
        public void onMatch(RelOptRuleCall call) {
            final MycatTransientSQLTableScan scan = call.rel(0);
            final RelOptTable table = scan.getTable();
            if (Bindables.BindableTableScan.canHandle(table)) {
                call.transformTo(
                        Bindables.BindableTableScan.create(scan.getCluster(), table));
            }
        }
    });
    final Program program = Programs.of(RuleSets.ofList(planner.getRules()));
    return program.run(planner, rel, rel.getTraitSet().replace(EnumerableConvention.INSTANCE),
            ImmutableList.of(), ImmutableList.of());
}
 

/** Test case for
 * <a href="https://issues.apache.org/jira/browse/CALCITE-2554">[CALCITE-2554]
 * Enrich enumerable join operators with order preserving information</a>.
 *
 * <p>Since join inputs are sorted, and this join preserves the order of the
 * left input, there shouldn't be any sort operator above the join.
 */
@Test void removeSortOverEnumerableSemiJoin() throws Exception {
  RuleSet prepareRules =
      RuleSets.ofList(
          SortProjectTransposeRule.INSTANCE,
          SemiJoinRule.PROJECT,
          SemiJoinRule.JOIN,
          EnumerableRules.ENUMERABLE_PROJECT_RULE,
          EnumerableRules.ENUMERABLE_SORT_RULE,
          EnumerableRules.ENUMERABLE_JOIN_RULE,
          EnumerableRules.ENUMERABLE_FILTER_RULE,
          EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE);
  String sql =
      "select e.\"deptno\" from \"hr\".\"emps\" e\n"
          + " where e.\"deptno\" in (select d.\"deptno\" from \"hr\".\"depts\" d)\n"
          + " order by e.\"empid\"";
  RelNode actualPlan = transform(sql, prepareRules);
  assertThat(
      toString(actualPlan),
      allOf(
          containsString("EnumerableHashJoin"),
          not(containsString("EnumerableSort"))));
}
 

/** Test case for
 * <a href="https://issues.apache.org/jira/browse/CALCITE-2554">[CALCITE-2554]
 * Enrich enumerable join operators with order preserving information</a>.
 *
 * <p>Since join inputs are sorted, and this join preserves the order of the
 * left input, there shouldn't be any sort operator above the join.
 *
 * <p>Until CALCITE-2018 is fixed we can add back EnumerableRules.ENUMERABLE_SORT_RULE
 */
@Test void removeSortOverEnumerableCorrelate() throws Exception {
  RuleSet prepareRules =
      RuleSets.ofList(
          SortProjectTransposeRule.INSTANCE,
          JoinToCorrelateRule.INSTANCE,
          EnumerableRules.ENUMERABLE_PROJECT_RULE,
          EnumerableRules.ENUMERABLE_CORRELATE_RULE,
          EnumerableRules.ENUMERABLE_FILTER_RULE,
          EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE);
  for (String joinType : Arrays.asList("left", "inner")) {
    String sql =
        "select e.\"deptno\" from \"hr\".\"emps\" e "
            + joinType + " join \"hr\".\"depts\" d "
            + " on e.\"deptno\" = d.\"deptno\" "
            + "order by e.\"empid\" ";
    RelNode actualPlan = transform(sql, prepareRules);
    assertThat(
        toString(actualPlan),
        allOf(
            containsString("EnumerableCorrelate"),
            not(containsString("EnumerableSort"))));
  }
}
 

/** Test case for
 * <a href="https://issues.apache.org/jira/browse/CALCITE-2554">[CALCITE-2554]
 * Enrich enumerable join operators with order preserving information</a>.
 *
 * <p>Since join inputs are sorted, and this join preserves the order of the
 * left input, there shouldn't be any sort operator above the join.
 */
@Test void removeSortOverEnumerableNestedLoopJoin() throws Exception {
  RuleSet prepareRules =
      RuleSets.ofList(
          SortProjectTransposeRule.INSTANCE,
          EnumerableRules.ENUMERABLE_JOIN_RULE,
          EnumerableRules.ENUMERABLE_PROJECT_RULE,
          EnumerableRules.ENUMERABLE_SORT_RULE,
          EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE);
  // Inner join is not considered since the ENUMERABLE_JOIN_RULE does not generate a nestedLoop
  // join in the case of inner joins.
  for (String joinType : Arrays.asList("left", "right", "full")) {
    String sql =
        "select e.\"deptno\" from \"hr\".\"emps\" e "
            + joinType + " join \"hr\".\"depts\" d "
            + " on e.\"deptno\" > d.\"deptno\" "
            + "order by e.\"empid\" ";
    RelNode actualPlan = transform(sql, prepareRules);
    assertThat(
        toString(actualPlan),
        allOf(
            containsString("EnumerableNestedLoopJoin"),
            not(containsString("EnumerableSort"))));
  }
}
 

/** Test case for
 * <a href="https://issues.apache.org/jira/browse/CALCITE-2554">[CALCITE-2554]
 * Enrich enumerable join operators with order preserving information</a>.
 *
 * <p>Since join inputs are sorted, and this join preserves the order of the
 * left input, there shouldn't be any sort operator above the join.
 */
@Test void removeSortOverEnumerableHashJoin() throws Exception {
  RuleSet prepareRules =
      RuleSets.ofList(
          SortProjectTransposeRule.INSTANCE,
          EnumerableRules.ENUMERABLE_JOIN_RULE,
          EnumerableRules.ENUMERABLE_PROJECT_RULE,
          EnumerableRules.ENUMERABLE_SORT_RULE,
          EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE);
  for (String joinType : Arrays.asList("left", "right", "full", "inner")) {
    String sql =
        "select e.\"deptno\" from \"hr\".\"emps\" e "
            + joinType + " join \"hr\".\"depts\" d "
            + " on e.\"deptno\" = d.\"deptno\" "
            + "order by e.\"empid\" ";
    RelNode actualPlan = transform(sql, prepareRules);
    assertThat(
        toString(actualPlan),
        allOf(
            containsString("EnumerableHashJoin"),
            not(containsString("EnumerableSort"))));
  }
}
 

private List<RelNode> optimizePlans(List<RelNode> originalRels,
    List<RelOptRule> rules) {
  final RelOptPlanner planner = originalRels.get(0).getCluster().getPlanner();
  // Remember old rule set of the planner before resetting it with new rules
  final List<RelOptRule> oldRules = planner.getRules();
  resetPlannerRules(planner, rules);
  final Program program = Programs.of(RuleSets.ofList(planner.getRules()));
  final List<RelNode> optimizedPlans = new ArrayList<>();
  for (RelNode rel : originalRels) {
    final RelCollation collation = rel instanceof Sort
        ? ((Sort) rel).collation
        : RelCollations.EMPTY;
    // Apply the planner to obtain the physical plan
    final RelNode physicalPlan = program.run(planner, rel,
        rel.getTraitSet().replace(EnumerableConvention.INSTANCE)
            .replace(collation).simplify(),
        ImmutableList.of(), ImmutableList.of());

    // Then convert the physical plan back to logical plan
    final RelNode logicalPlan = new ToLogicalConverter(builder).visit(physicalPlan);
    optimizedPlans.add(logicalPlan);
  }
  resetPlannerRules(planner, oldRules);
  return optimizedPlans;
}
 

@Override
public void setup() {
	util().buildBatchProgram(FlinkBatchProgram.DEFAULT_REWRITE());
	CalciteConfig calciteConfig = TableConfigUtils.getCalciteConfig(util().tableEnv().getConfig());
	calciteConfig.getBatchProgram().get().addLast(
			"rules",
			FlinkHepRuleSetProgramBuilder.<BatchOptimizeContext>newBuilder()
					.setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_SEQUENCE())
					.setHepMatchOrder(HepMatchOrder.BOTTOM_UP)
					.add(RuleSets.ofList(PushProjectIntoTableSourceScanRule.INSTANCE))
					.build()
	);

	String ddl1 =
			"CREATE TABLE MyTable (\n" +
					"  a int,\n" +
					"  b bigint,\n" +
					"  c string\n" +
					") WITH (\n" +
					" 'connector' = 'values',\n" +
					" 'bounded' = 'true'\n" +
					")";
	util().tableEnv().executeSql(ddl1);

	String ddl2 =
			"CREATE TABLE VirtualTable (\n" +
					"  a int,\n" +
					"  b bigint,\n" +
					"  c string,\n" +
					"  d as a + 1\n" +
					") WITH (\n" +
					" 'connector' = 'values',\n" +
					" 'bounded' = 'true'\n" +
					")";
	util().tableEnv().executeSql(ddl2);
}
 

@Test void testHintsPropagationInVolcanoPlannerRules() {
  final String sql = "select /*+ use_hash_join(r, s), use_hash_join(emp, dept) */\n"
      + "ename, job, sal, dept.name\n"
      + "from emp join dept on emp.deptno = dept.deptno";
  RelOptPlanner planner = new VolcanoPlanner();
  planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
  Tester tester1 = tester.withDecorrelation(true)
      .withClusterFactory(
        relOptCluster -> RelOptCluster.create(planner, relOptCluster.getRexBuilder()));
  final RelNode rel = tester1.convertSqlToRel(sql).rel;
  final RelHint hint = RelHint.builder("USE_HASH_JOIN")
      .inheritPath(0)
      .hintOption("EMP")
      .hintOption("DEPT")
      .build();
  // Validate Volcano planner.
  RuleSet ruleSet = RuleSets.ofList(
      new MockEnumerableJoinRule(hint), // Rule to validate the hint.
      FilterProjectTransposeRule.INSTANCE,
      FilterMergeRule.INSTANCE,
      ProjectMergeRule.INSTANCE,
      EnumerableRules.ENUMERABLE_JOIN_RULE,
      EnumerableRules.ENUMERABLE_PROJECT_RULE,
      EnumerableRules.ENUMERABLE_FILTER_RULE,
      EnumerableRules.ENUMERABLE_SORT_RULE,
      EnumerableRules.ENUMERABLE_LIMIT_RULE,
      EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE);
  Program program = Programs.of(ruleSet);
  RelTraitSet toTraits = rel
      .getCluster()
      .traitSet()
      .replace(EnumerableConvention.INSTANCE);

  program.run(planner, rel, toTraits,
      Collections.emptyList(), Collections.emptyList());
}
 

private static RelNode toPhysical(RelNode rel) {
  final RelOptPlanner planner = rel.getCluster().getPlanner();
  planner.clear();
  for (RelOptRule rule : RULE_SET) {
    planner.addRule(rule);
  }

  final Program program = Programs.of(RuleSets.ofList(planner.getRules()));
  return program.run(planner, rel, rel.getTraitSet().replace(EnumerableConvention.INSTANCE),
      ImmutableList.of(), ImmutableList.of());
}
 

@Test(expected = TableException.class)
public void testCustomCalciteConfig() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	CalciteConfig cc = new CalciteConfigBuilder()
			.replaceLogicalOptRuleSet(RuleSets.ofList())
			.replacePhysicalOptRuleSet(RuleSets.ofList())
			.build();
	tableEnv.getConfig().setCalciteConfig(cc);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table t = tableEnv.fromDataSet(ds);
	tableEnv.toDataSet(t, Row.class);
}
 

@Test(expected = TableException.class)
public void testCustomCalciteConfig() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	PlannerConfig cc = new CalciteConfigBuilder()
			.replaceLogicalOptRuleSet(RuleSets.ofList())
			.replacePhysicalOptRuleSet(RuleSets.ofList())
			.build();
	tableEnv.getConfig().setPlannerConfig(cc);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table t = tableEnv.fromDataSet(ds);
	tableEnv.toDataSet(t, Row.class);
}
 

public static RuleSet mergedRuleSets(RuleSet... ruleSets) {
  final ImmutableSet.Builder<RelOptRule> relOptRuleSetBuilder = ImmutableSet.builder();
  for (final RuleSet ruleSet : ruleSets) {
    relOptRuleSetBuilder.addAll(ruleSet);
  }
  return RuleSets.ofList(relOptRuleSetBuilder.build());
}
 

/**
 * Get the list of enabled reduce expression (logical) rules. These rules are enabled using session/system options.
 *
 * @param optimizerRulesContext used to get the list of planner settings, other rules may
 *                              also in the future need to get other query state from this,
 *                              such as the available list of UDFs (as is used by the
 *                              MergeProjectRule)
 * @return list of enabled reduce expression (logical) rules
 */
static RuleSet getEnabledReduceExpressionsRules(OptimizerRulesContext optimizerRulesContext) {
  final PlannerSettings ps = optimizerRulesContext.getPlannerSettings();

  // This list is used to store rules that can be turned on an off
  // by user facing planning options
  final ImmutableList.Builder<RelOptRule> userConfigurableRules = ImmutableList.builder();

  userConfigurableRules.add(ConvertCountDistinctToHll.INSTANCE);
  if (ps.options.getOption(PlannerSettings.REDUCE_ALGEBRAIC_EXPRESSIONS)) {
    userConfigurableRules.add(ReduceTrigFunctionsRule.INSTANCE);
  }

  if (ps.isConstantFoldingEnabled()) {
    // TODO - DRILL-2218, DX-2319
    if (ps.isReduceProjectExpressionsEnabled()) {
      userConfigurableRules.add(PROJECT_REDUCE_EXPRESSIONS_CALCITE_RULE);
    }
    if (ps.isReduceFilterExpressionsEnabled()) {
      userConfigurableRules.add(FILTER_REDUCE_EXPRESSIONS_CALCITE_RULE);
    }
    if (ps.isReduceCalcExpressionsEnabled()) {
      userConfigurableRules.add(CALC_REDUCE_EXPRESSIONS_CALCITE_RULE);
    }
  }
  return RuleSets.ofList(userConfigurableRules.build());
}
 

@Override
public RuleSet getStorageRules(OptimizerRulesContext context, PlannerPhase phase) {
  final ImmutableSet.Builder<RelOptRule> rules = ImmutableSet.builder();
  final Set<SourceType> types = new HashSet<>();

  try {
    for(ManagedStoragePlugin plugin : plugins.managed()){
      if(plugin.getState().getStatus() == SourceStatus.bad) {
        // we shouldn't consider rules for misbehaving plugins.
        continue;
      }
      final StoragePluginId pluginId = plugin.getId();

      StoragePluginRulesFactory factory = plugin.getRulesFactory();
      if(factory != null) {
        // add instance level rules.
        rules.addAll(factory.getRules(context, phase, pluginId));

        // add type level rules.
        if(types.add(pluginId.getType())) {
          rules.addAll(factory.getRules(context, phase, pluginId.getType()));
        }
      }
    }
  } catch (InstantiationException | IllegalAccessException e) {
    throw UserException.validationError(e).message("Failure getting plugin rules.").build(logger);
  }

  ImmutableSet<RelOptRule> rulesSet = rules.build();
  return RuleSets.ofList(rulesSet);
}
 

@Test(expected = TableException.class)
public void testCustomCalciteConfig() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	PlannerConfig cc = new CalciteConfigBuilder()
			.replaceLogicalOptRuleSet(RuleSets.ofList())
			.replacePhysicalOptRuleSet(RuleSets.ofList())
			.build();
	tableEnv.getConfig().setPlannerConfig(cc);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table t = tableEnv.fromDataSet(ds);
	tableEnv.toDataSet(t, Row.class);
}
 

public RuleSet getInjectedRules(PlannerPhase phase) {
  return RuleSets.ofList(sabotContext.getInjectedRulesFactories()
      .stream()
      .flatMap(rf -> rf.getRules(phase, optionManager).stream())
      .collect(Collectors.toList()));
}
 

static final RuleSet getPhysicalRules(OptimizerRulesContext optimizerRulesContext) {
  final List<RelOptRule> ruleList = new ArrayList<>();
  final PlannerSettings ps = optimizerRulesContext.getPlannerSettings();

  ruleList.add(SortConvertPrule.INSTANCE);
  ruleList.add(SortPrule.INSTANCE);
  ruleList.add(ProjectPrule.INSTANCE);
  ruleList.add(FlattenPrule.INSTANCE);
  ruleList.add(ScreenPrule.INSTANCE);
  ruleList.add(ExpandConversionRule.INSTANCE);
  ruleList.add(FilterPrule.INSTANCE);
  ruleList.add(LimitPrule.INSTANCE);
  ruleList.add(SamplePrule.INSTANCE);
  ruleList.add(SampleToLimitPrule.INSTANCE);
  ruleList.add(WriterPrule.INSTANCE);
  ruleList.add(WindowPrule.INSTANCE);
  ruleList.add(PushLimitToTopN.INSTANCE);
  ruleList.add(LimitUnionExchangeTransposeRule.INSTANCE);
  ruleList.add(UnionAllPrule.INSTANCE);
  ruleList.add(ValuesPrule.INSTANCE);
  ruleList.add(EmptyPrule.INSTANCE);
  ruleList.add(PushLimitToPruneableScan.INSTANCE);

  if (ps.isHashAggEnabled()) {
    ruleList.add(HashAggPrule.INSTANCE);
  }

  if (ps.isStreamAggEnabled()) {
    ruleList.add(StreamAggPrule.INSTANCE);
  }

  if (ps.isHashJoinEnabled()) {
    ruleList.add(HashJoinPrule.DIST_INSTANCE);

    if(ps.isBroadcastJoinEnabled()){
      ruleList.add(HashJoinPrule.BROADCAST_INSTANCE);
    }
  }

  if (ps.isMergeJoinEnabled()) {
    ruleList.add(MergeJoinPrule.DIST_INSTANCE);

    if(ps.isBroadcastJoinEnabled()){
      ruleList.add(MergeJoinPrule.BROADCAST_INSTANCE);
    }

  }

  // NLJ plans consist of broadcasting the right child, hence we need
  // broadcast join enabled.
  if (ps.isNestedLoopJoinEnabled() && ps.isBroadcastJoinEnabled()) {
    ruleList.add(NestedLoopJoinPrule.INSTANCE);
  }

  return RuleSets.ofList(ImmutableSet.copyOf(ruleList));
}
 

/** Test case for
 * <a href="https://issues.apache.org/jira/browse/CALCITE-2941">[CALCITE-2941]
 * EnumerableLimitRule on Sort with no collation creates EnumerableLimit with
 * wrong traitSet and cluster</a>.
 */
@Test void enumerableLimitOnEmptySort() throws Exception {
  RuleSet prepareRules =
      RuleSets.ofList(
          EnumerableRules.ENUMERABLE_FILTER_RULE,
          EnumerableRules.ENUMERABLE_SORT_RULE,
          EnumerableRules.ENUMERABLE_LIMIT_RULE,
          EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE);
  SchemaPlus rootSchema = Frameworks.createRootSchema(true);
  SchemaPlus defSchema = rootSchema.add("hr", new HrClusteredSchema());
  FrameworkConfig config = Frameworks.newConfigBuilder()
      .parserConfig(SqlParser.Config.DEFAULT)
      .defaultSchema(defSchema)
      .traitDefs(ConventionTraitDef.INSTANCE, RelCollationTraitDef.INSTANCE)
      .programs(Programs.of(prepareRules))
      .build();

  RelBuilder builder = RelBuilder.create(config);
  RelNode planBefore = builder
      .scan("hr", "emps")
      .sort(builder.field(0)) // will produce collation [0] in the plan
      .filter(
          builder.notEquals(
              builder.field(0),
              builder.literal(100)))
      .limit(1, 5) // force a limit inside an "empty" Sort (with no collation)
      .build();

  RelTraitSet desiredTraits = planBefore.getTraitSet()
      .replace(EnumerableConvention.INSTANCE);
  Program program = Programs.of(prepareRules);
  RelNode planAfter = program.run(planBefore.getCluster().getPlanner(), planBefore,
      desiredTraits, ImmutableList.of(), ImmutableList.of());

  // verify that the collation [0] is not lost in the final plan
  final RelCollation collation =
      planAfter.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE);
  assertThat(collation, notNullValue());
  final List<RelFieldCollation> fieldCollationList =
      collation.getFieldCollations();
  assertThat(fieldCollationList, notNullValue());
  assertThat(fieldCollationList.size(), is(1));
  assertThat(fieldCollationList.get(0).getFieldIndex(), is(0));
}