Java Code Examples for org.apache.calcite.rex.RexOver#containsOver()

/** As {@link #pushPastProject}, but returns null if the resulting expressions
 * are significantly more complex.
 *
 * @param bloat Maximum allowable increase in complexity */
public static @Nullable List<RexNode> pushPastProjectUnlessBloat(
    List<? extends RexNode> nodes, Project project, int bloat) {
  if (bloat < 0) {
    // If bloat is negative never merge.
    return null;
  }
  if (RexOver.containsOver(nodes, null)
      && RexOver.containsOver(project.getProjects(), null)) {
    // Is it valid relational algebra to apply windowed function to a windowed
    // function? Possibly. But it's invalid SQL, so don't go there.
    return null;
  }
  final List<RexNode> list = pushPastProject(nodes, project);
  final int bottomCount = RexUtil.nodeCount(project.getProjects());
  final int topCount = RexUtil.nodeCount(nodes);
  final int mergedCount = RexUtil.nodeCount(list);
  if (mergedCount > bottomCount + topCount + bloat) {
    // The merged expression is more complex than the input expressions.
    // Do not merge.
    return null;
  }
  return list;
}
 

@Override
public RelNode visit(LogicalProject project) {
	hasOverNode = RexOver.containsOver(project.getProjects(), null);
	final boolean hasSubQuery = RexUtil.SubQueryFinder.find(project.getProjects()) != null;
	try {
		if (!corNodeStack.isEmpty()) {
			mapSubQueryNodeToCorSet.put(project, corNodeStack.peek().getVariablesSet());
		}
		if (hasSubQuery) {
			corNodeStack.push(project);
		}
		checkCorCondition(project);
		for (RexNode node : project.getProjects()) {
			node.accept(rexVisitor(project));
		}
	} finally {
		if (hasSubQuery) {
			corNodeStack.pop();
		}
	}
	return super.visit(project);
}
 

@Override
public RelNode visit(LogicalProject project) {
	hasOverNode = RexOver.containsOver(project.getProjects(), null);
	final boolean hasSubQuery = RexUtil.SubQueryFinder.find(project.getProjects()) != null;
	try {
		if (!corNodeStack.isEmpty()) {
			mapSubQueryNodeToCorSet.put(project, corNodeStack.peek().getVariablesSet());
		}
		if (hasSubQuery) {
			corNodeStack.push(project);
		}
		checkCorCondition(project);
		for (RexNode node : project.getProjects()) {
			node.accept(rexVisitor(project));
		}
	} finally {
		if (hasSubQuery) {
			corNodeStack.pop();
		}
	}
	return super.visit(project);
}
 

public void onMatch(RelOptRuleCall call) {
  final Filter filter = call.rel(0);
  final Project project = call.rel(1);

  if (RexOver.containsOver(project.getProjects(), null)) {
    // In general a filter cannot be pushed below a windowing calculation.
    // Applying the filter before the aggregation function changes
    // the results of the windowing invocation.
    //
    // When the filter is on the PARTITION BY expression of the OVER clause
    // it can be pushed down. For now we don't support this.
    return;
  }
  // convert the filter to one that references the child of the project
  RexNode newCondition =
      RelOptUtil.pushPastProject(filter.getCondition(), project);

  final RelBuilder relBuilder = call.builder();
  RelNode newFilterRel;
  if (copyFilter) {
    newFilterRel = filter.copy(filter.getTraitSet(), project.getInput(),
        simplifyFilterCondition(newCondition, call));
  } else {
    newFilterRel =
        relBuilder.push(project.getInput()).filter(newCondition).build();
  }

  RelNode newProjRel =
      copyProject
          ? project.copy(project.getTraitSet(), newFilterRel,
              project.getProjects(), project.getRowType())
          : relBuilder.push(newFilterRel)
              .project(project.getProjects(), project.getRowType().getFieldNames())
              .build();

  call.transformTo(newProjRel);
}
 

/**
 * Since the project under aggregate maybe reduce expressions by {@link org.apache.kylin.query.optrule.AggregateProjectReduceRule},
 * consider the count of expressions into cost, the reduced project will be used.
 *
 * Made RexOver much more expensive so we can transform into {@link org.apache.kylin.query.relnode.OLAPWindowRel}
 * by rules in {@link org.apache.calcite.rel.rules.ProjectToWindowRule}
 */
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
    boolean hasRexOver = RexOver.containsOver(getProjects(), null);
    RelOptCost relOptCost = super.computeSelfCost(planner, mq).multiplyBy(.05)
            .multiplyBy(getProjects().size() * (double) (hasRexOver ? 50 : 1))
            .plus(planner.getCostFactory().makeCost(0.1 * caseCount, 0, 0));
    return planner.getCostFactory().makeCost(relOptCost.getRows(), 0, 0);
}
 

@Override
public boolean matches(RelOptRuleCall call) {
  FilterPrel filter = call.rel(0);
  ProjectPrel project = call.rel(1);
  if (RexOver.containsOver(project.getProjects(), null) ||
      RexUtil.containsCorrelation(filter.getCondition())) {
    return false;
  }

  NestedLoopJoinPrel join = call.rel(2);
  return join.getJoinType() == JoinRelType.INNER &&
      PrelUtil.getPlannerSettings(call.getPlanner()).getOptions().getOption(NestedLoopJoinPrel.VECTORIZED);
}
 

/**
 * Since the project under aggregate maybe reduce expressions by {@link org.apache.kylin.query.optrule.AggregateProjectReduceRule},
 * consider the count of expressions into cost, the reduced project will be used.
 *
 * Made RexOver much more expensive so we can transform into {@link org.apache.kylin.query.relnode.OLAPWindowRel}
 * by rules in {@link org.apache.calcite.rel.rules.ProjectToWindowRule}
 */
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
    boolean hasRexOver = RexOver.containsOver(getProjects(), null);
    RelOptCost relOptCost = super.computeSelfCost(planner, mq).multiplyBy(.05)
            .multiplyBy(getProjects().size() * (double) (hasRexOver ? 50 : 1))
            .plus(planner.getCostFactory().makeCost(0.1 * caseCount, 0, 0));
    return planner.getCostFactory().makeCost(relOptCost.getRows(), 0, 0);
}
 

/** Creates a JdbcProjectRule. */
public JdbcProjectRule(final JdbcConvention out,
    RelBuilderFactory relBuilderFactory) {
  super(Project.class, (Predicate<Project>) project ->
          (out.dialect.supportsWindowFunctions()
              || !RexOver.containsOver(project.getProjects(), null))
              && !userDefinedFunctionInProject(project),
      Convention.NONE, out, relBuilderFactory, "JdbcProjectRule");
}
 

/** Predicate for if a {@link Project} does not contain windowed aggregates. */
public static boolean notContainsWindowedAgg(Project project) {
  return !RexOver.containsOver(project.getProjects(), null);
}
 

/** Predicate for if a {@link Filter} does not windowed aggregates. */
public static boolean notContainsWindowedAgg(Filter filter) {
  return !RexOver.containsOver(filter.getCondition());
}
 

/** Returns whether a new sub-query is required. */
private boolean needNewSubQuery(RelNode rel, Clause[] clauses) {
  final Clause maxClause = maxClause();
  // If old and new clause are equal and belong to below set,
  // then new SELECT wrap is not required
  final Set<Clause> nonWrapSet = ImmutableSet.of(Clause.SELECT);
  for (Clause clause : clauses) {
    if (maxClause.ordinal() > clause.ordinal()
        || (maxClause == clause
            && !nonWrapSet.contains(clause))) {
      return true;
    }
  }

  if (rel instanceof Project
      && this.clauses.contains(Clause.HAVING)
      && dialect.getConformance().isHavingAlias()) {
    return true;
  }

  if (rel instanceof Project
      && RexOver.containsOver(((Project) rel).getProjects(), null)
      && maxClause == Clause.SELECT) {
    // Cannot merge a Project that contains windowed functions onto an
    // underlying Project
    return true;
  }

  if (rel instanceof Aggregate) {
    final Aggregate agg = (Aggregate) rel;
    final boolean hasNestedAgg = hasNestedAggregations(agg);
    if (!dialect.supportsNestedAggregations()
        && hasNestedAgg) {
      return true;
    }

    if (this.clauses.contains(Clause.GROUP_BY)) {
      // Avoid losing the distinct attribute of inner aggregate.
      return !hasNestedAgg || Aggregate.isNotGrandTotal(agg);
    }
  }

  return false;
}
 

public void onMatch(RelOptRuleCall call) {
  final Project origProject;
  final Filter filter;
  if (call.rels.length >= 2) {
    origProject = call.rel(0);
    filter = call.rel(1);
  } else {
    origProject = null;
    filter = call.rel(0);
  }
  final RelNode input = filter.getInput();
  final RexNode origFilter = filter.getCondition();

  if ((origProject != null)
      && RexOver.containsOver(origProject.getProjects(), null)) {
    // Cannot push project through filter if project contains a windowed
    // aggregate -- it will affect row counts. Abort this rule
    // invocation; pushdown will be considered after the windowed
    // aggregate has been implemented. It's OK if the filter contains a
    // windowed aggregate.
    return;
  }

  if ((origProject != null)
      && origProject.getRowType().isStruct()
      && origProject.getRowType().getFieldList().stream()
        .anyMatch(RelDataTypeField::isDynamicStar)) {
    // The PushProjector would change the plan:
    //
    //    prj(**=[$0])
    //    : - filter
    //        : - scan
    //
    // to form like:
    //
    //    prj(**=[$0])                    (1)
    //    : - filter                      (2)
    //        : - prj(**=[$0], ITEM= ...) (3)
    //            :  - scan
    // This new plan has more cost that the old one, because of the new
    // redundant project (3), if we also have FilterProjectTransposeRule in
    // the rule set, it will also trigger infinite match of the ProjectMergeRule
    // for project (1) and (3).
    return;
  }

  final RelBuilder builder = call.builder();
  final RelNode topProject;
  if (origProject != null && (wholeProject || wholeFilter)) {
    builder.push(input);

    final Set<RexNode> set = new LinkedHashSet<>();
    final RelOptUtil.InputFinder refCollector = new RelOptUtil.InputFinder();

    if (wholeFilter) {
      set.add(filter.getCondition());
    } else {
      filter.getCondition().accept(refCollector);
    }
    if (wholeProject) {
      set.addAll(origProject.getProjects());
    } else {
      refCollector.visitEach(origProject.getProjects());
    }

    // Build a list with inputRefs, in order, first, then other expressions.
    final List<RexNode> list = new ArrayList<>();
    final ImmutableBitSet refs = refCollector.build();
    for (RexNode field : builder.fields()) {
      if (refs.get(((RexInputRef) field).getIndex()) || set.contains(field)) {
        list.add(field);
      }
    }
    set.removeAll(list);
    list.addAll(set);
    builder.project(list);
    final Replacer replacer = new Replacer(list, builder);
    builder.filter(replacer.visit(filter.getCondition()));
    builder.project(replacer.visitList(origProject.getProjects()),
        origProject.getRowType().getFieldNames());
    topProject = builder.build();
  } else {
    // The traditional mode of operation of this rule: push down field
    // references. The effect is similar to RelFieldTrimmer.
    final PushProjector pushProjector =
        new PushProjector(origProject, origFilter, input,
            preserveExprCondition, builder);
    topProject = pushProjector.convertProject(null);
  }

  if (topProject != null) {
    call.transformTo(topProject);
  }
}
 

public void onMatch(RelOptRuleCall call) {
  LogicalProject origProj = call.rel(0);
  SetOp setOp = call.rel(1);

  // cannot push project past a distinct
  if (!setOp.all) {
    return;
  }

  // locate all fields referenced in the projection
  PushProjector pushProject =
      new PushProjector(
          origProj, null, setOp, preserveExprCondition, call.builder());
  pushProject.locateAllRefs();

  List<RelNode> newSetOpInputs = new ArrayList<>();
  int[] adjustments = pushProject.getAdjustments();

  final RelNode node;
  if (RexOver.containsOver(origProj.getProjects(), null)) {
    // should not push over past setop but can push its operand down.
    for (RelNode input : setOp.getInputs()) {
      Project p = pushProject.createProjectRefsAndExprs(input, true, false);
      // make sure that it is not a trivial project to avoid infinite loop.
      if (p.getRowType().equals(input.getRowType())) {
        return;
      }
      newSetOpInputs.add(p);
    }
    SetOp newSetOp =
        setOp.copy(setOp.getTraitSet(), newSetOpInputs);
    node = pushProject.createNewProject(newSetOp, adjustments);
  } else {
    // push some expressions below the setop; this
    // is different from pushing below a join, where we decompose
    // to try to keep expensive expressions above the join,
    // because UNION ALL does not have any filtering effect,
    // and it is the only operator this rule currently acts on
    setOp.getInputs().forEach(input ->
        newSetOpInputs.add(
            pushProject.createNewProject(
                pushProject.createProjectRefsAndExprs(
                    input, true, false), adjustments)));
    node = setOp.copy(setOp.getTraitSet(), newSetOpInputs);
  }

  call.transformTo(node);
}
 

public void onMatch(RelOptRuleCall call) {
  final LogicalProject project = call.rel(0);
  final LogicalCalc calc = call.rel(1);

  // Don't merge a project which contains windowed aggregates onto a
  // calc. That would effectively be pushing a windowed aggregate down
  // through a filter. Transform the project into an identical calc,
  // which we'll have chance to merge later, after the over is
  // expanded.
  final RelOptCluster cluster = project.getCluster();
  RexProgram program =
      RexProgram.create(
          calc.getRowType(),
          project.getProjects(),
          null,
          project.getRowType(),
          cluster.getRexBuilder());
  if (RexOver.containsOver(program)) {
    LogicalCalc projectAsCalc = LogicalCalc.create(calc, program);
    call.transformTo(projectAsCalc);
    return;
  }

  // Create a program containing the project node's expressions.
  final RexBuilder rexBuilder = cluster.getRexBuilder();
  final RexProgramBuilder progBuilder =
      new RexProgramBuilder(
          calc.getRowType(),
          rexBuilder);
  for (Pair<RexNode, String> field : project.getNamedProjects()) {
    progBuilder.addProject(field.left, field.right);
  }
  RexProgram topProgram = progBuilder.getProgram();
  RexProgram bottomProgram = calc.getProgram();

  // Merge the programs together.
  RexProgram mergedProgram =
      RexProgramBuilder.mergePrograms(
          topProgram,
          bottomProgram,
          rexBuilder);
  final LogicalCalc newCalc =
      LogicalCalc.create(calc.getInput(), mergedProgram);
  call.transformTo(newCalc);
}
 

public void onMatch(RelOptRuleCall call) {
  final Filter filter = call.rel(0);
  final Project project = call.rel(1);

  if (RexOver.containsOver(project.getProjects(), null)) {
    // In general a filter cannot be pushed below a windowing calculation.
    // Applying the filter before the aggregation function changes
    // the results of the windowing invocation.
    //
    // When the filter is on the PARTITION BY expression of the OVER clause
    // it can be pushed down. For now we don't support this.
    return;
  }
  // convert the filter to one that references the child of the project
  RexNode newCondition =
      RelOptUtil.pushPastProject(filter.getCondition(), project);

  final RelBuilder relBuilder = call.builder();
  RelNode newFilterRel;
  if (copyFilter) {
    final RelNode input = project.getInput();
    final RelTraitSet traitSet = filter.getTraitSet()
        .replaceIfs(RelCollationTraitDef.INSTANCE,
            () -> Collections.singletonList(
                    input.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE)))
        .replaceIfs(RelDistributionTraitDef.INSTANCE,
            () -> Collections.singletonList(
                    input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE)));
    newCondition = RexUtil.removeNullabilityCast(relBuilder.getTypeFactory(), newCondition);
    newFilterRel = filter.copy(traitSet, input, newCondition);
  } else {
    newFilterRel =
        relBuilder.push(project.getInput()).filter(newCondition).build();
  }

  RelNode newProjRel =
      copyProject
          ? project.copy(project.getTraitSet(), newFilterRel,
              project.getProjects(), project.getRowType())
          : relBuilder.push(newFilterRel)
              .project(project.getProjects(), project.getRowType().getFieldNames())
              .build();

  call.transformTo(newProjRel);
}
 

private static boolean noWindowFunc(Project project) {
    return !RexOver.containsOver(project.getProjects(), null);
}
 

/** Predicate for whether a {@link Project} contains multisets or windowed
 * aggregates. */
public static boolean containsMultisetOrWindowedAgg(Project project) {
    return !(B && RexMultisetUtil.containsMultiset(project.getProjects(), true)
            || RexOver.containsOver(project.getProjects(), null));
}
 

/** Predicate for whether a {@link Filter} contains multisets or windowed
 * aggregates. */
public static boolean containsMultisetOrWindowedAgg(Filter filter) {
    return !(B && RexMultisetUtil.containsMultiset(filter.getCondition(), true)
            || RexOver.containsOver(filter.getCondition()));
}
 

public void onMatch(RelOptRuleCall call) {
  final LogicalProject project = call.rel(0);
  final LogicalCalc calc = call.rel(1);

  // Don't merge a project which contains windowed aggregates onto a
  // calc. That would effectively be pushing a windowed aggregate down
  // through a filter. Transform the project into an identical calc,
  // which we'll have chance to merge later, after the over is
  // expanded.
  final RelOptCluster cluster = project.getCluster();
  RexProgram program =
      RexProgram.create(
          calc.getRowType(),
          project.getProjects(),
          null,
          project.getRowType(),
          cluster.getRexBuilder());
  if (RexOver.containsOver(program)) {
    LogicalCalc projectAsCalc = LogicalCalc.create(calc, program);
    call.transformTo(projectAsCalc);
    return;
  }

  // Create a program containing the project node's expressions.
  final RexBuilder rexBuilder = cluster.getRexBuilder();
  final RexProgramBuilder progBuilder =
      new RexProgramBuilder(
          calc.getRowType(),
          rexBuilder);
  for (Pair<RexNode, String> field : project.getNamedProjects()) {
    progBuilder.addProject(field.left, field.right);
  }
  RexProgram topProgram = progBuilder.getProgram();
  RexProgram bottomProgram = calc.getProgram();

  // Merge the programs together.
  RexProgram mergedProgram =
      RexProgramBuilder.mergePrograms(
          topProgram,
          bottomProgram,
          rexBuilder);
  final LogicalCalc newCalc =
      LogicalCalc.create(calc.getInput(), mergedProgram);
  call.transformTo(newCalc);
}