org.apache.calcite.rel.rules.ProjectRemoveRule Java Examples

public void registerAbstractRelationalRules() {
  addRule(FilterJoinRule.FILTER_ON_JOIN);
  addRule(FilterJoinRule.JOIN);
  addRule(AbstractConverter.ExpandConversionRule.INSTANCE);
  addRule(JoinCommuteRule.INSTANCE);
  addRule(SemiJoinRule.PROJECT);
  addRule(SemiJoinRule.JOIN);
  if (CalciteSystemProperty.COMMUTE.value()) {
    addRule(JoinAssociateRule.INSTANCE);
  }
  addRule(AggregateRemoveRule.INSTANCE);
  addRule(UnionToDistinctRule.INSTANCE);
  addRule(ProjectRemoveRule.INSTANCE);
  addRule(AggregateJoinTransposeRule.INSTANCE);
  addRule(AggregateProjectMergeRule.INSTANCE);
  addRule(CalcRemoveRule.INSTANCE);
  addRule(SortRemoveRule.INSTANCE);

  // todo: rule which makes Project({OrdinalRef}) disappear
}
 

/**
 * Converts a relational expression to a form where
 * {@link org.apache.calcite.rel.logical.LogicalJoin}s are
 * as close to leaves as possible.
 */
public static RelNode toLeafJoinForm(RelNode rel) {
  final Program program = Programs.hep(
      ImmutableList.of(
          JoinProjectTransposeRule.RIGHT_PROJECT,
          JoinProjectTransposeRule.LEFT_PROJECT,
          FilterJoinRule.FilterIntoJoinRule.FILTER_ON_JOIN,
          ProjectRemoveRule.INSTANCE,
          ProjectMergeRule.INSTANCE),
      false,
      DefaultRelMetadataProvider.INSTANCE);
  if (CalciteSystemProperty.DEBUG.value()) {
    System.out.println(
        RelOptUtil.dumpPlan("before", rel, SqlExplainFormat.TEXT,
            SqlExplainLevel.DIGEST_ATTRIBUTES));
  }
  final RelNode rel2 = program.run(null, rel, null,
      ImmutableList.of(),
      ImmutableList.of());
  if (CalciteSystemProperty.DEBUG.value()) {
    System.out.println(
        RelOptUtil.dumpPlan("after", rel2, SqlExplainFormat.TEXT,
            SqlExplainLevel.DIGEST_ATTRIBUTES));
  }
  return rel2;
}
 

private RelNode canonicalize(RelNode rel) {
  HepProgram program =
      new HepProgramBuilder()
          .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
          .addRuleInstance(FilterMergeRule.INSTANCE)
          .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
          .addRuleInstance(FilterJoinRule.JOIN)
          .addRuleInstance(FilterAggregateTransposeRule.INSTANCE)
          .addRuleInstance(ProjectMergeRule.INSTANCE)
          .addRuleInstance(ProjectRemoveRule.INSTANCE)
          .addRuleInstance(ProjectJoinTransposeRule.INSTANCE)
          .addRuleInstance(ProjectSetOpTransposeRule.INSTANCE)
          .addRuleInstance(FilterToCalcRule.INSTANCE)
          .addRuleInstance(ProjectToCalcRule.INSTANCE)
          .addRuleInstance(FilterCalcMergeRule.INSTANCE)
          .addRuleInstance(ProjectCalcMergeRule.INSTANCE)
          .addRuleInstance(CalcMergeRule.INSTANCE)
          .build();
  final HepPlanner hepPlanner = new HepPlanner(program);
  hepPlanner.setRoot(rel);
  return hepPlanner.findBestExp();
}
 

/**
 * Converts a relational expression to a form where
 * {@link org.apache.calcite.rel.logical.LogicalJoin}s are
 * as close to leaves as possible.
 */
public static RelNode toLeafJoinForm(RelNode rel) {
  final Program program = Programs.hep(
      ImmutableList.of(
          JoinProjectTransposeRule.RIGHT_PROJECT,
          JoinProjectTransposeRule.LEFT_PROJECT,
          FilterJoinRule.FilterIntoJoinRule.FILTER_ON_JOIN,
          ProjectRemoveRule.INSTANCE,
          ProjectMergeRule.INSTANCE),
      false,
      DefaultRelMetadataProvider.INSTANCE);
  if (CalciteSystemProperty.DEBUG.value()) {
    System.out.println(
        RelOptUtil.dumpPlan("before", rel, SqlExplainFormat.TEXT,
            SqlExplainLevel.DIGEST_ATTRIBUTES));
  }
  final RelNode rel2 = program.run(null, rel, null,
      ImmutableList.of(),
      ImmutableList.of());
  if (CalciteSystemProperty.DEBUG.value()) {
    System.out.println(
        RelOptUtil.dumpPlan("after", rel2, SqlExplainFormat.TEXT,
            SqlExplainLevel.DIGEST_ATTRIBUTES));
  }
  return rel2;
}
 

@Override
public void onMatch(RelOptRuleCall call) {
  final Project proj = call.rel(0);
  final ScanCrel scan = call.rel(1);

  ProjectPushInfo columnInfo = PrelUtil.getColumns(scan.getRowType(), proj.getProjects());

  // get TableBase, either wrapped in RelOptTable, or TranslatableTable. TableBase table = scan.getTable().unwrap(TableBase.class);
  if (columnInfo == null || columnInfo.isStarQuery()) {
    return;
  }

  ScanCrel newScan = scan.cloneWithProject(columnInfo.columns);

  List<RexNode> newProjects = Lists.newArrayList();
  for (RexNode n : proj.getChildExps()) {
    newProjects.add(n.accept(columnInfo.getInputRewriter()));
  }

  final RelBuilder relBuilder = relBuilderFactory.create(proj.getCluster(), null);
  relBuilder.push(newScan);
  relBuilder.project(newProjects, proj.getRowType().getFieldNames());
  final RelNode newProj = relBuilder.build();

  if (newProj instanceof Project
      && ProjectRemoveRule.isTrivial((Project) newProj)
      && newScan.getRowType().getFullTypeString().equals(newProj.getRowType().getFullTypeString())) {
      call.transformTo(newScan);
  } else {
    if(newScan.getProjectedColumns().equals(scan.getProjectedColumns())) {
      // no point in doing a pushdown that doesn't change anything.
      return;
    }

    call.transformTo(newProj);
  }
}
 

@Test void testReplaceCommonSubexpression() throws Exception {
  // Note that here it may look like the rule is firing
  // twice, but actually it's only firing once on the
  // common sub-expression.  The purpose of this test
  // is to make sure the planner can deal with
  // rewriting something used as a common sub-expression
  // twice by the same parent (the join in this case).

  final String sql = "select d1.deptno from (select * from dept) d1,\n"
      + "(select * from dept) d2";
  sql(sql).withRule(ProjectRemoveRule.INSTANCE).check();
}
 

@Override public void register(RelOptPlanner planner) {
  for (RelOptRule rule : JdbcRules.rules(this)) {
    planner.addRule(rule);
  }
  planner.addRule(FilterSetOpTransposeRule.INSTANCE);
  planner.addRule(ProjectRemoveRule.INSTANCE);
}
 

public static boolean isTrivialProject(Project project, boolean useNamesInIdentityProjCalc) {
  if (!useNamesInIdentityProjCalc) {
    return ProjectRemoveRule.isTrivial(project);
  }  else {
    return containIdentity(project.getProjects(), project.getRowType(), project.getInput().getRowType());
  }
}
 

private static List<RelNode> substitute(
    RelNode root, RelOptMaterialization materialization) {
  // First, if the materialization is in terms of a star table, rewrite
  // the query in terms of the star table.
  if (materialization.starTable != null) {
    RelNode newRoot = RelOptMaterialization.tryUseStar(root,
        materialization.starRelOptTable);
    if (newRoot != null) {
      root = newRoot;
    }
  }

  // Push filters to the bottom, and combine projects on top.
  RelNode target = materialization.queryRel;
  HepProgram program =
      new HepProgramBuilder()
          .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
          .addRuleInstance(ProjectMergeRule.INSTANCE)
          .addRuleInstance(ProjectRemoveRule.INSTANCE)
          .build();

  final HepPlanner hepPlanner = new HepPlanner(program);
  hepPlanner.setRoot(target);
  target = hepPlanner.findBestExp();

  hepPlanner.setRoot(root);
  root = hepPlanner.findBestExp();

  return new MaterializedViewSubstitutionVisitor(target, root)
          .go(materialization.tableRel);
}
 

@Override
public void onMatch(RelOptRuleCall call) {
  final ProjectForFlattenRel proj = call.rel(0);
  final ScanRelBase scan = call.rel(1);

  try {
    final ProjectPushInfo columnInfoItemsExprs = PrelUtil.getColumns(scan.getRowType(), proj.getItemExprs());
    if (columnInfoItemsExprs == null || columnInfoItemsExprs.isStarQuery()) {
      return;
    }

    final ScanRelBase newScan = scan.cloneWithProject(columnInfoItemsExprs.columns);

    // if the scan is the same as this one (no change in projections), no need to push down.
    if(newScan.getProjectedColumns().equals(scan.getProjectedColumns())){
      return;
    }

    List<RexNode> newProjects = Lists.newArrayList();
    for (RexNode n : proj.getProjExprs()) {
      newProjects.add(n.accept(columnInfoItemsExprs.getInputRewriter()));
    }

    final ProjectRel newProj =
            ProjectRel.create(proj.getCluster(),
                    proj.getTraitSet().plus(Rel.LOGICAL),
                    newScan,
                    newProjects,
                    proj.getRowType());

    if (ProjectRemoveRule.isTrivial(newProj)) {
      call.transformTo(newScan);
    } else {
      call.transformTo(newProj);
    }
  } catch (Exception e) {
    throw Throwables.propagate(e);
  }
}
 

public static boolean isTrivialProject(Project project, boolean useNamesInIdentityProjCalc) {
    if (!useNamesInIdentityProjCalc) {
        return ProjectRemoveRule.isTrivial(project);
    } else {
        return containIdentity(project.getProjects(), project.getRowType(), project.getInput().getRowType());
    }
}
 

@Override
public Prel visitProject(ProjectPrel prel, Void value) throws RuntimeException {
  // Require prefix rename : there exists other expression, in addition to a star column.
  if (!prefixedForStar  // not set yet.
      && StarColumnHelper.containsStarColumnInProject(prel.getInput().getRowType(), prel.getProjects())
      && prel.getRowType().getFieldNames().size() > 1) {
    prefixedForStar = true;
  }

  // For project, we need make sure that the project's field name is same as the input,
  // when the project expression is RexInPutRef, since we may insert a PAS which will
  // rename the projected fields.

  RelNode child = ((Prel) prel.getInput(0)).accept(this, null);

  List<String> fieldNames = Lists.newArrayList();

  for (Pair<String, RexNode> pair : Pair.zip(prel.getRowType().getFieldNames(), prel.getProjects())) {
    if (pair.right instanceof RexInputRef) {
      String name = child.getRowType().getFieldNames().get(((RexInputRef) pair.right).getIndex());
      fieldNames.add(name);
    } else {
      fieldNames.add(pair.left);
    }
  }

  // Make sure the field names are unique : no allow of duplicate field names in a rowType.
  fieldNames = makeUniqueNames(fieldNames);

  RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(),
      prel.getProjects(), fieldNames, null);

  ProjectPrel newProj = (ProjectPrel) prel.copy(prel.getTraitSet(), child, prel.getProjects(), rowType);

  if (ProjectRemoveRule.isTrivial(newProj)) {
    return (Prel) child;
  } else {
    return newProj;
  }
}
 

@Override
public void onMatch(RelOptRuleCall call) {
    DrillProjectRel projectRel = call.rel(0);
    DrillScanRel scanRel = call.rel(1);

    ItemStarFieldsVisitor itemStarFieldsVisitor = new ItemStarFieldsVisitor(
            scanRel.getRowType().getFieldNames());
    List<RexNode> projects = projectRel.getProjects();
    for (RexNode project : projects) {
        project.accept(itemStarFieldsVisitor);
    }

    // if there are no item fields, no need to proceed further
    if (itemStarFieldsVisitor.hasNoItemStarFields()) {
        return;
    }

    Map<String, DesiredField> itemStarFields = itemStarFieldsVisitor.getItemStarFields();

    DrillScanRel newScan = createNewScan(scanRel, itemStarFields);

    // re-write projects
    Map<RexNode, Integer> fieldMapper = createFieldMapper(itemStarFields.values(),
            scanRel.getRowType().getFieldCount());
    FieldsReWriter fieldsReWriter = new FieldsReWriter(fieldMapper);
    List<RexNode> newProjects = new ArrayList<>();
    for (RexNode node : projectRel.getChildExps()) {
        newProjects.add(node.accept(fieldsReWriter));
    }

    DrillProjectRel newProject = new DrillProjectRel(projectRel.getCluster(), projectRel.getTraitSet(), newScan,
            newProjects, projectRel.getRowType());

    if (ProjectRemoveRule.isTrivial(newProject)) {
        call.transformTo(newScan);
    } else {
        call.transformTo(newProject);
    }
}
 

@Override
public void onMatch(RelOptRuleCall call) {
  Project project = call.rel(0);
  TableScan scan = call.rel(1);

  try {
    if (scan.getRowType().getFieldList().isEmpty()) {
      return;
    }

    ProjectPushInfo projectPushInfo = DrillRelOptUtil.getFieldsInformation(scan.getRowType(), project.getProjects());
    if (!canPushProjectIntoScan(scan.getTable(), projectPushInfo)
        || skipScanConversion(projectPushInfo.createNewRowType(project.getCluster().getTypeFactory()), scan)) {
      // project above scan may be removed in ProjectRemoveRule for the case when it is trivial
      return;
    }

    DrillScanRelBase newScan = createScan(scan, projectPushInfo);

    List<RexNode> newProjects = new ArrayList<>();
    for (RexNode n : project.getChildExps()) {
      newProjects.add(n.accept(projectPushInfo.getInputReWriter()));
    }

    DrillProjectRelBase newProject =
        createProject(project, newScan, newProjects);

    if (ProjectRemoveRule.isTrivial(newProject)) {
      call.transformTo(newScan);
    } else {
      call.transformTo(newProject);
    }
  } catch (IOException e) {
    throw new DrillRuntimeException(e);
  }
}
 

protected HepProgramBuilder getProgramBuilder() {
  return new HepProgramBuilder()
    .addRuleInstance(PushFilterPastProjectRule.CALCITE_NO_CHILD_CHECK)
    .addRuleInstance(ProjectMergeRule.INSTANCE)
    .addRuleInstance(ProjectRemoveRule.INSTANCE);
}
 

@Override
public Prel visitProject(ProjectPrel prel, Void value) throws RuntimeException {
  ProjectPrel proj = prel;

  // Require prefix rename : there exists other expression, in addition to a star column.
  if (!prefixedForStar  // not set yet.
      && StarColumnHelper.containsStarColumnInProject(prel.getInput().getRowType(), proj.getProjects())
      && prel.getRowType().getFieldNames().size() > 1) {
    prefixedForStar = true;
  }

  // For project, we need make sure that the project's field name is same as the input,
  // when the project expression is RexInPutRef, since we may insert a PAS which will
  // rename the projected fields.

  RelNode child = ((Prel) prel.getInput(0)).accept(this, null);

  List<String> fieldNames = Lists.newArrayList();

  for (Pair<String, RexNode> pair : Pair.zip(prel.getRowType().getFieldNames(), proj.getProjects())) {
    if (pair.right instanceof RexInputRef) {
      String name = child.getRowType().getFieldNames().get(((RexInputRef) pair.right).getIndex());
      fieldNames.add(name);
    } else {
      fieldNames.add(pair.left);
    }
  }

  // Make sure the field names are unique : no allow of duplicate field names in a rowType.
  fieldNames = makeUniqueNames(fieldNames);

  RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), proj.getProjects(), fieldNames);

  ProjectPrel newProj = (ProjectPrel) proj.copy(proj.getTraitSet(), child, proj.getProjects(), rowType);

  if (ProjectRemoveRule.isTrivial(newProj)) {
    return (Prel) child;
  } else {
    return newProj;
  }
}
 

@Override
public void register(RelOptPlanner planner) {
    planner.addRule(FilterSetOpTransposeRule.INSTANCE);
    planner.addRule(ProjectRemoveRule.INSTANCE);
}
 

@Override
public void onMatch(RelOptRuleCall call) {
	LogicalProject project = call.rel(0);
	LogicalTableScan scan = call.rel(1);

	int[] usedFields = RexNodeExtractor.extractRefInputFields(project.getProjects());
	// if no fields can be projected, we keep the original plan.
	if (scan.getRowType().getFieldCount() == usedFields.length) {
		return;
	}

	TableSourceTable oldTableSourceTable = scan.getTable().unwrap(TableSourceTable.class);
	DynamicTableSource newTableSource = oldTableSourceTable.tableSource().copy();

	int[][] projectedFields = new int[usedFields.length][];
	List<String> fieldNames = new ArrayList<>();
	for (int i = 0; i < usedFields.length; ++i) {
		int usedField = usedFields[i];
		projectedFields[i] = new int[] { usedField };
		fieldNames.add(scan.getRowType().getFieldNames().get(usedField));
	}
	((SupportsProjectionPushDown) newTableSource).applyProjection(projectedFields);
	FlinkTypeFactory flinkTypeFactory = (FlinkTypeFactory) oldTableSourceTable.getRelOptSchema().getTypeFactory();
	RelDataType newRowType = flinkTypeFactory.projectStructType(oldTableSourceTable.getRowType(), usedFields);

	// project push down does not change the statistic, we can reuse origin statistic
	TableSourceTable newTableSourceTable = oldTableSourceTable.copy(
			newTableSource, newRowType, new String[] { ("project=[" + String.join(", ", fieldNames) + "]") });

	LogicalTableScan newScan = new LogicalTableScan(
			scan.getCluster(), scan.getTraitSet(), scan.getHints(), newTableSourceTable);
	// rewrite input field in projections
	List<RexNode> newProjects = RexNodeRewriter.rewriteWithNewFieldInput(project.getProjects(), usedFields);
	LogicalProject newProject = project.copy(
			project.getTraitSet(),
			newScan,
			newProjects,
			project.getRowType());

	if (ProjectRemoveRule.isTrivial(newProject)) {
		// drop project if the transformed program merely returns its input
		call.transformTo(newScan);
	} else {
		call.transformTo(newProject);
	}
}
 

private static List<RelNode> substitute(
    RelNode root, RelOptMaterialization materialization) {
  // First, if the materialization is in terms of a star table, rewrite
  // the query in terms of the star table.
  if (materialization.starTable != null) {
    RelNode newRoot = RelOptMaterialization.tryUseStar(root,
        materialization.starRelOptTable);
    if (newRoot != null) {
      root = newRoot;
    }
  }

  // Push filters to the bottom, and combine projects on top.
  RelNode target = materialization.queryRel;
  // try to trim unused field in relational expressions.
  root = trimUnusedfields(root);
  target = trimUnusedfields(target);
  HepProgram program =
      new HepProgramBuilder()
          .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
          .addRuleInstance(FilterMergeRule.INSTANCE)
          .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
          .addRuleInstance(FilterJoinRule.JOIN)
          .addRuleInstance(FilterAggregateTransposeRule.INSTANCE)
          .addRuleInstance(ProjectMergeRule.INSTANCE)
          .addRuleInstance(ProjectRemoveRule.INSTANCE)
          .addRuleInstance(ProjectJoinTransposeRule.INSTANCE)
          .addRuleInstance(ProjectSetOpTransposeRule.INSTANCE)
          .addRuleInstance(FilterToCalcRule.INSTANCE)
          .addRuleInstance(ProjectToCalcRule.INSTANCE)
          .addRuleInstance(FilterCalcMergeRule.INSTANCE)
          .addRuleInstance(ProjectCalcMergeRule.INSTANCE)
          .addRuleInstance(CalcMergeRule.INSTANCE)
          .build();

  // We must use the same HEP planner for the two optimizations below.
  // Thus different nodes with the same digest will share the same vertex in
  // the plan graph. This is important for the matching process.
  final HepPlanner hepPlanner = new HepPlanner(program);
  hepPlanner.setRoot(target);
  target = hepPlanner.findBestExp();

  hepPlanner.setRoot(root);
  root = hepPlanner.findBestExp();

  return new SubstitutionVisitor(target, root).go(materialization.tableRel);
}
 

private void removeTrivialProject(boolean useRule) {
  VolcanoPlanner planner = new VolcanoPlanner();

  planner.addRelTraitDef(ConventionTraitDef.INSTANCE);

  if (useRule) {
    planner.addRule(ProjectRemoveRule.INSTANCE);
  }

  planner.addRule(new PhysLeafRule());
  planner.addRule(new GoodSingleRule());
  planner.addRule(new PhysProjectRule());

  planner.addRule(
      new ConverterRule(
          RelNode.class,
          PHYS_CALLING_CONVENTION,
          EnumerableConvention.INSTANCE,
          "PhysToIteratorRule") {
        public RelNode convert(RelNode rel) {
          return new PhysToIteratorConverter(
              rel.getCluster(),
              rel);
        }
      });

  RelOptCluster cluster = newCluster(planner);
  PhysLeafRel leafRel =
      new PhysLeafRel(
          cluster,
          "a");
  final RelBuilder relBuilder =
      RelFactories.LOGICAL_BUILDER.create(leafRel.getCluster(), null);
  RelNode projectRel =
      relBuilder.push(leafRel)
          .project(relBuilder.alias(relBuilder.field(0), "this"))
          .build();
  NoneSingleRel singleRel =
      new NoneSingleRel(
          cluster,
          projectRel);
  RelNode convertedRel =
      planner.changeTraits(
          singleRel,
          cluster.traitSetOf(EnumerableConvention.INSTANCE));
  planner.setRoot(convertedRel);
  RelNode result = planner.chooseDelegate().findBestExp();
  assertTrue(result instanceof PhysToIteratorConverter);
}