org.apache.calcite.rel.RelShuttleImpl Java Examples

private RelRoot expandView(RelOptTable.ToRelContext context,
    RelDataType rowType, String queryString) {
  try {
    final RelRoot root =
        context.expandView(rowType, queryString, schemaPath, viewPath);
    final RelNode rel = RelOptUtil.createCastRel(root.rel, rowType, true);
    // Expand any views
    final RelNode rel2 = rel.accept(
        new RelShuttleImpl() {
          @Override public RelNode visit(TableScan scan) {
            final RelOptTable table = scan.getTable();
            final TranslatableTable translatableTable =
                table.unwrap(TranslatableTable.class);
            if (translatableTable != null) {
              return translatableTable.toRel(context, table);
            }
            return super.visit(scan);
          }
        });
    return root.withRel(rel2);
  } catch (Exception e) {
    throw new RuntimeException("Error while parsing view definition: "
        + queryString, e);
  }
}
 

public DatasourceInfo preComputeSeq(RelNode relNode) {
    MycatDBContext uponDBContext = dataContext.getUponDBContext();
    Map<String, List<SingeTargetSQLTable>> map = new HashMap<>();
    relNode.accept(new RelShuttleImpl() {
        @Override
        public RelNode visit(TableScan scan) {
            SingeTargetSQLTable unwrap = scan.getTable().unwrap(SingeTargetSQLTable.class);
            if (unwrap != null) {
                List<SingeTargetSQLTable> preComputationSQLTables = map.computeIfAbsent(uponDBContext.resolveFinalTargetName(unwrap.getTargetName()), s -> new ArrayList<>(1));
                preComputationSQLTables.add(unwrap);
            }
            return super.visit(scan);
        }
    });
    List<SingeTargetSQLTable> preSeq = new ArrayList<>();
    for (Map.Entry<String, List<SingeTargetSQLTable>> stringListEntry : map.entrySet()) {
        List<SingeTargetSQLTable> value = stringListEntry.getValue();
        int size = value.size() - 1;
        for (int i = 0; i < size; i++) {
            preSeq.add(value.get(i));
        }
    }
    return new DatasourceInfo(preSeq, map);
}
 

private RelRoot expandView(RelOptTable.ToRelContext context,
    RelDataType rowType, String queryString) {
  try {
    final RelRoot root =
        context.expandView(rowType, queryString, schemaPath, viewPath);
    final RelNode rel = RelOptUtil.createCastRel(root.rel, rowType, true);
    // Expand any views
    final RelNode rel2 = rel.accept(
        new RelShuttleImpl() {
          @Override public RelNode visit(TableScan scan) {
            final RelOptTable table = scan.getTable();
            final TranslatableTable translatableTable =
                table.unwrap(TranslatableTable.class);
            if (translatableTable != null) {
              return translatableTable.toRel(context, table);
            }
            return super.visit(scan);
          }
        });
    return root.withRel(rel2);
  } catch (Exception e) {
    throw new RuntimeException("Error while parsing view definition: "
        + queryString, e);
  }
}
 

public static RelNode removeParentExpansionNodes(NamespaceKey pathFilter, RelNode node) {
  RelNode rel = node.accept(new RelShuttleImpl() {
    @Override
    public RelNode visit(RelNode other) {
      if(other instanceof ExpansionNode) {
        ExpansionNode e = (ExpansionNode) other;
        if(e.getPath().equals(pathFilter)) {
          return e.copy(e.getTraitSet(), e.getInputs());
        } else {
          RelNode input = e.getInput().accept(this);
          if (input == e.getInput()) {
            return e;
          } else {
            return input;
          }
        }
      }
      return super.visit(other);
    }
  });
  return rel;
}
 

public static RelNode removeAllButRoot(RelNode tree) {

    return tree.accept(new RelShuttleImpl() {

      private boolean alreadyFound = false;

      @Override
      public RelNode visit(RelNode other) {
        if(other instanceof ExpansionNode) {
          if(alreadyFound) {
            return ((ExpansionNode) other).getInput();
          } else {
            alreadyFound = true;
          }
        }
        return super.visit(other);
      }});
  }
 

@Override
public void planConvertedToRel(RelNode converted, long millisTaken) {
  final String convertedRelTree = toStringOrEmpty(converted, true);

  try {
    RelNode toSerialize = converted.accept(new RelShuttleImpl() {
      @Override
      public RelNode visit(TableScan scan) {
        return LogicalTableScan.create(scan.getCluster(), scan.getTable());
      }
    });

    serializedPlan = relSerializerFactory.getSerializer(converted.getCluster()).serializeToBytes(toSerialize);
  } catch (Throwable e) {
    logger.debug("Error", e);
  }

  planPhases.add(PlanPhaseProfile.newBuilder()
    .setPhaseName("Convert To Rel")
    .setDurationMillis(millisTaken)
    .setPlan(convertedRelTree)
    .build());
}
 

@Override
public void onMatch(RelOptRuleCall call) {

  RelNode result = doMatch(call);

  if (result == null) {
    return;
  }

  call.transformTo(result.accept(new RelShuttleImpl() {
    @Override
    public RelNode visit(LogicalJoin join) {
      TransformCollectingCall c2 = new TransformCollectingCall(call.getPlanner(), JoinPushTransitivePredicatesRule.INSTANCE.getOperand(), new RelNode[] {join}, null);
      JoinPushTransitivePredicatesRule.INSTANCE.onMatch(c2);
      if (c2.outcome.isEmpty()) {
        return join;
      } else {
        return c2.outcome.get(0);
      }
    }
  }));
}
 

/** Returns the schema of a {@link org.apache.calcite.rel.core.TableScan}
 * in this plan, or null if there are no scans. */
private RelOptSchema getSchema(RelNode rel) {
  final Holder<RelOptSchema> schemaHolder = Holder.of(null);
  rel.accept(
      new RelShuttleImpl() {
        @Override public RelNode visit(TableScan scan) {
          schemaHolder.set(scan.getTable().getRelOptSchema());
          return super.visit(scan);
        }
      });
  return schemaHolder.get();
}
 

@Test void testRelShuttleForLogicalTableModify() {
  final String sql = "insert into emp select * from emp";
  final LogicalTableModify rel = (LogicalTableModify) tester.convertSqlToRel(sql).rel;
  final List<RelNode> rels = new ArrayList<>();
  final RelShuttleImpl visitor = new RelShuttleImpl() {
    @Override public RelNode visit(LogicalTableModify modify) {
      RelNode visitedRel = super.visit(modify);
      rels.add(visitedRel);
      return visitedRel;
    }
  };
  visitor.visit(rel);
  assertThat(rels.size(), is(1));
  assertThat(rels.get(0), isA(LogicalTableModify.class));
}
 

public static void foo(RelNode rel) {
  // Convert rel tree to JSON.
  final RelJsonWriter writer = new RelJsonWriter();
  rel.explain(writer);
  final String json = writer.asString();

  // Find the schema. If there are no tables in the plan, we won't need one.
  final RelOptSchema[] schemas = {null};
  rel.accept(new RelShuttleImpl() {
    @Override public RelNode visit(TableScan scan) {
      schemas[0] = scan.getTable().getRelOptSchema();
      return super.visit(scan);
    }
  });

  // Convert JSON back to rel tree.
  Frameworks.withPlanner((cluster, relOptSchema, rootSchema) -> {
    final RelJsonReader reader = new RelJsonReader(
        cluster,
        schemas[0], rootSchema);
    try {
      RelNode x = reader.read(json);
    } catch (IOException e) {
      throw TestUtil.rethrow(e);
    }
    return null;
  });
}
 

private RelNode expand(RelOptTable.ToRelContext context) {
	final RelNode rel = convertToRel(context);
	// Expand any views
	return rel.accept(
		new RelShuttleImpl() {
			@Override
			public RelNode visit(TableScan scan) {
				final RelOptTable table = scan.getTable();
				if (table instanceof ExpandingPreparingTable) {
					return ((ExpandingPreparingTable) table).expand(context);
				}
				return super.visit(scan);
			}
		});
}
 

/**
 * check whether a node has some input which have correlation condition.
 * e.g. SELECT * FROM l WHERE EXISTS (SELECT * FROM r LEFT JOIN (SELECT * FROM t WHERE t.j=l.b) t1 ON r.f=t1.k)
 * the above sql can not be converted to semi-join plan,
 * because the right input of Left-Join has the correlation condition(t.j=l.b).
 */
private void checkCorConditionOfInput(final RelNode input) {
	final RelShuttleImpl shuttle = new RelShuttleImpl() {
		final RexVisitor<Void> visitor = new RexVisitorImpl<Void>(true) {
			@Override
			public Void visitCorrelVariable(RexCorrelVariable correlVariable) {
				hasUnsupportedCorCondition = true;
				return super.visitCorrelVariable(correlVariable);
			}
		};

		@Override
		public RelNode visit(LogicalFilter filter) {
			filter.getCondition().accept(visitor);
			return super.visit(filter);
		}

		@Override
		public RelNode visit(LogicalProject project) {
			for (RexNode rex : project.getProjects()) {
				rex.accept(visitor);
			}
			return super.visit(project);
		}

		@Override
		public RelNode visit(LogicalJoin join) {
			join.getCondition().accept(visitor);
			return super.visit(join);
		}
	};
	input.accept(shuttle);
}
 

public static RelNode removeFromTree(RelNode tree) {
  return tree.accept(new RelShuttleImpl() {

    @Override
    public RelNode visit(RelNode other) {
      if(other instanceof ExpansionNode) {
        return ((ExpansionNode) super.visit(other)).getInput();
      }
      return super.visit(other);
    }});
}
 

public RelNode applyStrippedNodes(RelNode belowStrippedNode) {
  return stripFragment.accept(new RelShuttleImpl() {

    @Override
    public RelNode visit(RelNode other) {
      if(other instanceof StripLeaf) {
        return belowStrippedNode;
      }
      return super.visit(other);
    }

  });
}
 

public RelNode convertToMycatRel(RelNode relNode) {
    return relNode.accept(new RelShuttleImpl() {
        @Override
        public RelNode visit(TableScan scan) {
            MycatSQLTableScan unwrap = scan.getTable().unwrap(MycatSQLTableScan.class);
            if (unwrap != null) {
                return unwrap.toRel(ViewExpanders.toRelContext(MycatCalcitePlanner.this, MycatCalcitePlanner.this.newCluster()), scan.getTable());
            }
            return super.visit(scan);
        }
    });
}
 

/**
 * check whether a node has some input which have correlation condition.
 * e.g. SELECT * FROM l WHERE EXISTS (SELECT * FROM r LEFT JOIN (SELECT * FROM t WHERE t.j=l.b) t1 ON r.f=t1.k)
 * the above sql can not be converted to semi-join plan,
 * because the right input of Left-Join has the correlation condition(t.j=l.b).
 */
private void checkCorConditionOfInput(final RelNode input) {
	final RelShuttleImpl shuttle = new RelShuttleImpl() {
		final RexVisitor<Void> visitor = new RexVisitorImpl<Void>(true) {
			@Override
			public Void visitCorrelVariable(RexCorrelVariable correlVariable) {
				hasUnsupportedCorCondition = true;
				return super.visitCorrelVariable(correlVariable);
			}
		};

		@Override
		public RelNode visit(LogicalFilter filter) {
			filter.getCondition().accept(visitor);
			return super.visit(filter);
		}

		@Override
		public RelNode visit(LogicalProject project) {
			for (RexNode rex : project.getProjects()) {
				rex.accept(visitor);
			}
			return super.visit(project);
		}

		@Override
		public RelNode visit(LogicalJoin join) {
			join.getCondition().accept(visitor);
			return super.visit(join);
		}
	};
	input.accept(shuttle);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  // normalize join first
  RelNode normalized = JoinNormalizationRule.INSTANCE.normalize(this);
  if (normalized != this) {
    // If normalized, return sum of all converted/generated rels
    final Pointer<RelOptCost> cost = new Pointer<>(planner.getCostFactory().makeZeroCost());
    normalized.accept(new RelShuttleImpl() {
      @Override
      public RelNode visit(RelNode other) {
        cost.value = cost.value.plus(mq.getNonCumulativeCost(other));
        if (!(other instanceof Join)) {
          return super.visit(other);
        }
        return other;
      }
    });
    return cost.value;
  }

  // Compute filter cost
  RelOptCost remainingFilterCost;
  if (remaining.isAlwaysTrue()) {
    remainingFilterCost = planner.getCostFactory().makeZeroCost();
  } else {
    // Similar to FilterRelBase
    double inputRows = Math.max(mq.getRowCount(getLeft()), mq.getRowCount(getRight()));
    double compNum = inputRows;
    double rowCompNum = this.getRowType().getFieldCount() * inputRows ;

    final List<RexNode> conjunctions = RelOptUtil.conjunctions(condition);
    final int conjunctionsSize = conjunctions.size();
    for (int i = 0; i< conjunctionsSize; i++) {
      RexNode conjFilter = RexUtil.composeConjunction(this.getCluster().getRexBuilder(), conjunctions.subList(0, i + 1), false);
      compNum += RelMdUtil.estimateFilteredRows(this, conjFilter, mq);
    }

    double cpuCost = compNum * DremioCost.COMPARE_CPU_COST + rowCompNum * DremioCost.COPY_COST;
    Factory costFactory = (Factory)planner.getCostFactory();
    // Do not include input rows into the extra filter cost
    remainingFilterCost = costFactory.makeCost(0, cpuCost, 0, 0);
  }
  return remainingFilterCost.plus(doComputeSelfCost(planner, mq));
}
 

/**
 *  Given a relNode tree for SELECT statement, convert to Dremio Logical RelNode tree.
 * @param relNode
 * @return
 * @throws SqlUnsupportedException
 * @throws RelConversionException
 */
public static Rel convertToDrel(SqlHandlerConfig config, final RelNode relNode) throws SqlUnsupportedException, RelConversionException {

  try {
    final RelNode trimmed = trimFields(relNode, true, config.getContext().getPlannerSettings().isRelPlanningEnabled());
    final RelNode preLog = transform(config, PlannerType.HEP_AC, PlannerPhase.PRE_LOGICAL, trimmed, trimmed.getTraitSet(), true);

    final RelTraitSet logicalTraits = preLog.getTraitSet().plus(Rel.LOGICAL);
    final RelNode adjusted = transform(config, PlannerType.VOLCANO, PlannerPhase.LOGICAL, preLog, logicalTraits, true);

    final Catalog catalog = config.getContext().getCatalog();
    if (catalog instanceof CachingCatalog) {
      config.getObserver().tablesCollected(catalog.getAllRequestedTables());
    }

    final RelNode intermediateNode;
    if (config.getContext().getPlannerSettings().removeRowCountAdjustment()) {
      intermediateNode = adjusted.accept(new RelShuttleImpl() {
          @Override
          public RelNode visit(TableScan scan) {
            if (scan instanceof FilesystemScanDrel) {
              FilesystemScanDrel scanDrel = (FilesystemScanDrel) scan;
              return new FilesystemScanDrel(
                scanDrel.getCluster(),
                scanDrel.getTraitSet(),
                scanDrel.getTable(),
                scanDrel.getPluginId(),
                scanDrel.getTableMetadata(),
                scanDrel.getProjectedColumns(),
                1.0);
            }
            return super.visit(scan);
          }
        });
    } else {
      intermediateNode = adjusted;
    }

    RelNode postLogical;
    if (config.getContext().getPlannerSettings().isRelPlanningEnabled()) {
      final RelNode decorrelatedNode = DremioRelDecorrelator.decorrelateQuery(intermediateNode, DremioRelFactories.LOGICAL_BUILDER.create(intermediateNode.getCluster(), null), true, true);
      final RelNode jdbcPushDown = transform(config, PlannerType.HEP_AC, PlannerPhase.RELATIONAL_PLANNING, decorrelatedNode, decorrelatedNode.getTraitSet().plus(Rel.LOGICAL), true);
      postLogical = jdbcPushDown.accept(new ShortenJdbcColumnAliases()).accept(new ConvertJdbcLogicalToJdbcRel(DremioRelFactories.LOGICAL_BUILDER));
    } else {
      postLogical = intermediateNode;
    }

    // Do Join Planning.
    final RelNode preConvertedRelNode = transform(config, PlannerType.HEP_BOTTOM_UP, PlannerPhase.JOIN_PLANNING_MULTI_JOIN, postLogical, postLogical.getTraitSet(), true);
    final RelNode convertedRelNode = transform(config, PlannerType.HEP_BOTTOM_UP, PlannerPhase.JOIN_PLANNING_OPTIMIZATION, preConvertedRelNode, preConvertedRelNode.getTraitSet(), true);

    FlattenRelFinder flattenFinder = new FlattenRelFinder();
    final RelNode flattendPushed;
    if (flattenFinder.run(convertedRelNode)) {
      flattendPushed = transform(config, PlannerType.VOLCANO, PlannerPhase.FLATTEN_PUSHDOWN,
        convertedRelNode, convertedRelNode.getTraitSet(), true);
    } else {
      flattendPushed = convertedRelNode;
    }

    final Rel drel = (Rel) flattendPushed;

    if (drel instanceof TableModify) {
      throw new UnsupportedOperationException("TableModify " + drel);
    } else {
      final Optional<SubstitutionInfo> acceleration = findUsedMaterializations(config, drel);
      if (acceleration.isPresent()) {
        config.getObserver().planAccelerated(acceleration.get());
      }
      return drel;
    }
  } catch (RelOptPlanner.CannotPlanException ex) {
    logger.error(ex.getMessage(), ex);

    if(JoinUtils.checkCartesianJoin(relNode, Lists.<Integer>newArrayList(), Lists.<Integer>newArrayList(), Lists.<Boolean>newArrayList())) {
      throw new UnsupportedRelOperatorException("This query cannot be planned\u2014possibly due to use of an unsupported feature.");
    } else {
      throw ex;
    }
  }
}
 

@SneakyThrows
    public static RowBaseIterator run(String sql, MycatCalciteDataContext calciteDataContext, RelNode relNode) {
        SqlRecorder recorder = SqlRecorderRuntime.INSTANCE.getCurrentRecorder();
        Map<String, List<SingeTargetSQLTable>> map = new HashMap<>();
        relNode.accept(new RelShuttleImpl() {
            @Override
            public RelNode visit(TableScan scan) {
                SingeTargetSQLTable unwrap = scan.getTable().unwrap(SingeTargetSQLTable.class);
                if (unwrap != null && !unwrap.existsEnumerable()) {
                    List<SingeTargetSQLTable> tables = map.computeIfAbsent(unwrap.getTargetName(), s -> new ArrayList<>(2));
                    tables.add(unwrap);
                }
                return super.visit(scan);
            }
        });

        HepProgramBuilder hepProgramBuilder = new HepProgramBuilder();
        hepProgramBuilder.addMatchLimit(64);

        hepProgramBuilder.addRuleInstance(StreamUnionRule.INSTANCE);
        final HepPlanner planner2 = new HepPlanner(hepProgramBuilder.build());
        planner2.setRoot(relNode);
        relNode = planner2.findBestExp();

        //check
        relNode.accept(new RelShuttleImpl() {
            @Override
            public RelNode visit(LogicalUnion union) {
                if (union.getInputs().size() > 2) {
                    throw new AssertionError("union input more 2");
                }
                return super.visit(union);
            }
        });
        long startGetConnectionTime = TimeProvider.INSTANCE.now();
        fork(sql, calciteDataContext, map);
        long cbo = TimeProvider.INSTANCE.now();
        recorder.addRecord(SqlRecorderType.GET_CONNECTION, sql, cbo - startGetConnectionTime);
        ArrayBindable bindable1 = Interpreters.bindable(relNode);
        long execution_start = TimeProvider.INSTANCE.now();
        recorder.addRecord(SqlRecorderType.CBO, sql, execution_start - cbo);
//        EnumerableInterpretable.toBindable()
        Enumerable<Object[]> bind = bindable1.bind(calciteDataContext);
        Enumerator<Object[]> enumerator = bind.enumerator();

        return new EnumeratorRowIterator(CalciteConvertors.getMycatRowMetaData(relNode.getRowType()), enumerator,
                () -> {
                    recorder.addRecord(SqlRecorderType.EXECUTION_TIME, sql, TimeProvider.INSTANCE.now()-execution_start);
                    recorder.addRecord(SqlRecorderType.AT_END, sql, TimeProvider.INSTANCE.now());
                });
    }
 

/** Test case for
 * <a href="https://issues.apache.org/jira/browse/CALCITE-3183">[CALCITE-3183]
 * Trimming method for Filter rel uses wrong traitSet</a>. */
@Test void testFilterAndSortWithTrim() {
  // Create a customized test with RelCollation trait in the test cluster.
  Tester tester = new TesterImpl(getDiffRepos(),
      false, true,
      true, false, true,
      null, null) {
    @Override public RelOptPlanner createPlanner() {
      return new MockRelOptPlanner(Contexts.empty()) {
        @Override public List<RelTraitDef> getRelTraitDefs() {
          return ImmutableList.<RelTraitDef>of(RelCollationTraitDef.INSTANCE);
        }
        @Override public RelTraitSet emptyTraitSet() {
          return RelTraitSet.createEmpty().plus(
              RelCollationTraitDef.INSTANCE.getDefault());
        }
      };
    }
  };

  // Run query and save plan after trimming
  final String sql = "select count(a.EMPNO)\n"
      + "from (select * from emp order by sal limit 3) a\n"
      + "where a.EMPNO > 10 group by 2";
  RelNode afterTrim = tester.convertSqlToRel(sql).rel;

  // Get Sort and Filter operators
  final List<RelNode> rels = new ArrayList<>();
  final RelShuttleImpl visitor = new RelShuttleImpl() {
    @Override public RelNode visit(LogicalSort sort) {
      rels.add(sort);
      return super.visit(sort);
    }
    @Override public RelNode visit(LogicalFilter filter) {
      rels.add(filter);
      return super.visit(filter);
    }
  };
  visitor.visit(afterTrim);

  // Ensure sort and filter operators have consistent traitSet after trimming
  assertThat(rels.size(), is(2));
  RelTrait filterCollation = rels.get(0).getTraitSet()
      .getTrait(RelCollationTraitDef.INSTANCE);
  RelTrait sortCollation = rels.get(1).getTraitSet()
      .getTrait(RelCollationTraitDef.INSTANCE);
  assertTrue(filterCollation.satisfies(sortCollation));
}