Java Code Examples for org.apache.calcite.rel.type.RelDataType#getField()

/**
 * Find a join condition's left input source scan Prel. If we can't find a target scan Prel then this
 * RuntimeFilter can not pushed down to a probe side scan Prel.
 *
 * @param fieldName   left join condition field Name
 * @param leftRelNode left RelNode of a BiRel or the SingleRel
 * @return a left scan Prel which contains the left join condition name or null
 */
private ScanPrel findLeftScanPrel(String fieldName, RelNode leftRelNode) {
  if (leftRelNode instanceof ScanPrel) {
    RelDataType scanRowType = leftRelNode.getRowType();
    RelDataTypeField field = scanRowType.getField(fieldName, true, true);
    if (field != null) {
      //found
      return (ScanPrel) leftRelNode;
    } else {
      return null;
    }
  } else if (leftRelNode instanceof RelSubset) {
    RelNode bestNode = ((RelSubset) leftRelNode).getBest();
    if (bestNode != null) {
      return findLeftScanPrel(fieldName, bestNode);
    } else {
      return null;
    }
  } else {
    List<RelNode> relNodes = leftRelNode.getInputs();
    RelNode leftNode = relNodes.get(0);
    return findLeftScanPrel(fieldName, leftNode);
  }
}
 

@Override
public RelNode visit(LogicalProject project) {
  RelNode input = project.getInput().accept(this);
  RelDataType incomingRowType = input.getRowType();
  List<RexNode> newProjects;
  RelDataTypeField modField = incomingRowType.getField(UPDATE_COLUMN, false, false);
  if (modField == null) {
    return project;
  }
  newProjects = FluentIterable.from(project.getProjects())
    .append(new RexInputRef(modField.getIndex(), modField.getType()))
    .toList();
  FieldInfoBuilder fieldInfoBuilder = new FieldInfoBuilder(project.getCluster().getTypeFactory());
  for (RelDataTypeField field : project.getRowType().getFieldList()) {
    fieldInfoBuilder.add(field);
  }
  fieldInfoBuilder.add(UPDATE_COLUMN, modField.getType());
  return new LogicalProject(
    project.getCluster(),
    project.getTraitSet(),
    input,
    newProjects,
    fieldInfoBuilder.build()
  );
}
 

@Override
public RelNode visit(LogicalAggregate aggregate) {
  RelNode input = aggregate.getInput().accept(this);
  RelDataType incomingRowType = input.getRowType();
  RelDataTypeField modField = incomingRowType.getField(UPDATE_COLUMN, false, false);
  if (modField == null) {
    return aggregate;
  }

  final AggregateCall aggCall = AggregateCall.create(SqlStdOperatorTable.MAX, false, ImmutableList.of(modField.getIndex()), -1, modField.getType(), UPDATE_COLUMN);
  final List<AggregateCall> aggCalls = FluentIterable.from(aggregate.getAggCallList())
    .append(aggCall)
    .toList();
  return aggregate.copy(
    aggregate.getTraitSet(),
    input,
    aggregate.indicator,
    aggregate.getGroupSet(),
    null,
    aggCalls
  );
}
 

/**
 * Gets a map of indexes from the source to fields in the target for the
 * intersecting set of source and target fields.
 *
 * @param sourceFields The source of column names that determine indexes
 * @param targetFields The target fields to be indexed
 */
public static ImmutableMap<Integer, RelDataTypeField> getIndexToFieldMap(
    List<RelDataTypeField> sourceFields,
    RelDataType targetFields) {
  final ImmutableMap.Builder<Integer, RelDataTypeField> output =
      ImmutableMap.builder();
  for (final RelDataTypeField source : sourceFields) {
    final RelDataTypeField target = targetFields.getField(source.getName(), true, false);
    if (target != null) {
      output.put(source.getIndex(), target);
    }
  }
  return output.build();
}
 

public static RelDataType createTypeFromProjection(RelDataType type,
    List<String> columnNameList, RelDataTypeFactory typeFactory,
    boolean caseSensitive) {
  // If the names in columnNameList and type have case-sensitive differences,
  // the resulting type will use those from type. These are presumably more
  // canonical.
  final List<RelDataTypeField> fields =
      new ArrayList<>(columnNameList.size());
  for (String name : columnNameList) {
    RelDataTypeField field = type.getField(name, caseSensitive, false);
    fields.add(type.getFieldList().get(field.getIndex()));
  }
  return typeFactory.createStructType(fields);
}
 

@Override public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
  final RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
  final RelDataType operandType = opBinding.getOperandType(0);
  switch (operandType.getSqlTypeName()) {
  case ARRAY:
    return typeFactory.createTypeWithNullability(
        operandType.getComponentType(), true);
  case MAP:
    return typeFactory.createTypeWithNullability(operandType.getValueType(),
        true);
  case ROW:
    String fieldName = opBinding.getOperandLiteralValue(1, String.class);
    RelDataTypeField field = operandType.getField(fieldName, false, false);
    if (field == null) {
      throw new AssertionError("Cannot infer type of field '"
          + fieldName + "' within ROW type: " + operandType);
    } else {
      return field.getType();
    }
  case ANY:
  case DYNAMIC_STAR:
    return typeFactory.createTypeWithNullability(
        typeFactory.createSqlType(SqlTypeName.ANY), true);
  default:
    throw new AssertionError();
  }
}
 

private List<Integer> resolvePartitionKeys(){
  final List<Integer> keys = Lists.newArrayList();
  final RelDataType inputRowType = getInput().getRowType();
  final List<String> partitionCol = getCreateTableEntry().getPartitionColumns();

  for (final String col : partitionCol) {
    final RelDataTypeField field = inputRowType.getField(col, false, false);
    Preconditions.checkArgument(field != null,
        String.format("partition col %s could not be resolved in table's column lists!", col));
    keys.add(field.getIndex());
  }

  return keys;
}
 

public static RelDataType createTypeFromProjection(RelDataType type,
    List<String> columnNameList, RelDataTypeFactory typeFactory,
    boolean caseSensitive) {
  // If the names in columnNameList and type have case-sensitive differences,
  // the resulting type will use those from type. These are presumably more
  // canonical.
  final List<RelDataTypeField> fields =
      new ArrayList<>(columnNameList.size());
  for (String name : columnNameList) {
    RelDataTypeField field = type.getField(name, caseSensitive, false);
    fields.add(type.getFieldList().get(field.getIndex()));
  }
  return typeFactory.createStructType(fields);
}
 

/**
 * Gets a map of indexes from the source to fields in the target for the
 * intersecting set of source and target fields.
 *
 * @param sourceFields The source of column names that determine indexes
 * @param targetFields The target fields to be indexed
 */
public static ImmutableMap<Integer, RelDataTypeField> getIndexToFieldMap(
    List<RelDataTypeField> sourceFields,
    RelDataType targetFields) {
  final ImmutableMap.Builder<Integer, RelDataTypeField> output =
      ImmutableMap.builder();
  for (final RelDataTypeField source : sourceFields) {
    final RelDataTypeField target = targetFields.getField(source.getName(), true, false);
    if (target != null) {
      output.put(source.getIndex(), target);
    }
  }
  return output.build();
}
 

@Deprecated // to be removed before 2.0
public static RelDataTypeField lookupField(boolean caseSensitive,
    final RelDataType rowType, String columnName) {
  return rowType.getField(columnName, caseSensitive, false);
}
 

public RelDataTypeField field(RelDataType rowType, String fieldName) {
  return rowType.getField(fieldName, caseSensitive, false);
}
 

@Deprecated // to be removed before 2.0
public static RelDataTypeField lookupField(boolean caseSensitive,
    final RelDataType rowType, String columnName) {
  return rowType.getField(columnName, caseSensitive, false);
}
 

private void verifyRowtimeAttribute(
		Map<FeatureOption, MergingStrategy> mergingStrategies,
		SqlIdentifier eventTimeColumnName,
		Map<String, RelDataType> allFieldsTypes) {
	String fullRowtimeExpression = eventTimeColumnName.toString();
	boolean specAlreadyExists = watermarkSpecs.containsKey(fullRowtimeExpression);

	if (specAlreadyExists &&
		mergingStrategies.get(FeatureOption.WATERMARKS) != MergingStrategy.OVERWRITING) {
		throw new ValidationException(String.format(
			"There already exists a watermark spec for column '%s' in the base table. You " +
				"might want to specify EXCLUDING WATERMARKS or OVERWRITING WATERMARKS.",
			fullRowtimeExpression));
	}

	List<String> components = eventTimeColumnName.names;
	if (!allFieldsTypes.containsKey(components.get(0))) {
		throw new ValidationException(
			String.format(
				"The rowtime attribute field '%s' is not defined in the table schema, at %s\n" +
					"Available fields: [%s]",
				fullRowtimeExpression,
				eventTimeColumnName.getParserPosition(),
				allFieldsTypes.keySet().stream().collect(Collectors.joining("', '", "'", "'"))
			));
	}

	if (components.size() > 1) {
		RelDataType componentType = allFieldsTypes.get(components.get(0));
		for (int i = 1; i < components.size(); i++) {
			RelDataTypeField field = componentType.getField(components.get(i), true, false);
			if (field == null) {
				throw new ValidationException(
					String.format(
						"The rowtime attribute field '%s' is not defined in the table schema, at %s\n" +
							"Nested field '%s' was not found in a composite type: %s.",
						fullRowtimeExpression,
						eventTimeColumnName.getComponent(i).getParserPosition(),
						components.get(i),
						FlinkTypeFactory.toLogicalType(allFieldsTypes.get(components.get(0))))
					);
			}
			componentType = field.getType();
		}
	}
}
 

@Override
public RelNode visit(LogicalAggregate aggregate) {
  RelNode input = aggregate.getInput().accept(this);

  // Create a new project with null UPDATE_COLUMN below aggregate
  final RelBuilder relBuilder = newCalciteRelBuilderWithoutContext(aggregate.getCluster());
  relBuilder.push(input);
  List<RexNode> nodes = input.getRowType().getFieldList().stream().map(q -> {
    if (UPDATE_COLUMN.equals(q.getName())) {
      return relBuilder.getRexBuilder().makeNullLiteral(q.getType());
    } else{
      return relBuilder.getRexBuilder().makeInputRef(q.getType(), q.getIndex());
    }
  }).collect(Collectors.toList());
  relBuilder.project(nodes, input.getRowType().getFieldNames());

  // create a new aggregate with null UPDATE_COLUMN in groupSet
  RelDataType incomingRowType = relBuilder.peek().getRowType();
  RelDataTypeField modField = incomingRowType.getField(UPDATE_COLUMN, false, false);
  ImmutableBitSet newGroupSet = aggregate.getGroupSet().rebuild().set(modField.getIndex()).build();
  GroupKey groupKey = relBuilder.groupKey(newGroupSet, aggregate.indicator, null);

  final int groupCount = aggregate.getGroupCount();
  final Pointer<Integer> ind = new Pointer<>(groupCount-1);
  final List<String> fieldNames = aggregate.getRowType().getFieldNames();
  final List<AggregateCall> aggCalls = aggregate.getAggCallList().stream().map(q -> {
    ind.value++;
    if (q.getName() == null) {
      return q.rename(fieldNames.get(ind.value));
    }
    return q;
  }).collect(Collectors.toList());

  relBuilder.aggregate(groupKey, aggCalls);

  // create a new project on top to preserve rowType
  Iterable<RexInputRef> projects = FluentIterable.from(aggregate.getRowType().getFieldNames())
    .transform(new Function<String, RexInputRef>() {
      @Override
      public RexInputRef apply(String fieldName) {
        return relBuilder.field(fieldName);
      }
    })
    .append(relBuilder.field(UPDATE_COLUMN));

  relBuilder.project(projects);

  return relBuilder.build();
}
 

@Override
public RelDataTypeField field(RelDataType rowType, String columnName) {
  return rowType.getField(columnName, false, false);
}
 

/**
 * Generate a possible RuntimeFilter of a HashJoinPrel, left some BF parameters of the generated RuntimeFilter
 * to be set later.
 *
 * @param hashJoinPrel
 * @return null or a partial information RuntimeFilterDef
 */
private RuntimeFilterDef generateRuntimeFilter(HashJoinPrel hashJoinPrel) {
  JoinRelType joinRelType = hashJoinPrel.getJoinType();
  JoinInfo joinInfo = hashJoinPrel.analyzeCondition();
  boolean allowJoin = (joinInfo.isEqui()) && (joinRelType == JoinRelType.INNER || joinRelType == JoinRelType.RIGHT);
  if (!allowJoin) {
    return null;
  }
  //TODO check whether to enable RuntimeFilter according to the NDV percent
  /**
   double threshold = 0.5;
   double percent = leftNDV / rightDNV;
   if (percent > threshold ) {
   return null;
   }
   */

  List<BloomFilterDef> bloomFilterDefs = new ArrayList<>();
  //find the possible left scan node of the left join key
  ScanPrel probeSideScanPrel = null;
  RelNode left = hashJoinPrel.getLeft();
  RelNode right = hashJoinPrel.getRight();
  ExchangePrel exchangePrel = findRightExchangePrel(right);
  if (exchangePrel == null) {
    //Does not support the single fragment mode ,that is the right build side
    //can only be BroadcastExchangePrel or HashToRandomExchangePrel
    return null;
  }
  List<String> leftFields = left.getRowType().getFieldNames();
  List<String> rightFields = right.getRowType().getFieldNames();
  List<Integer> leftKeys = hashJoinPrel.getLeftKeys();
  List<Integer> rightKeys = hashJoinPrel.getRightKeys();
  RelMetadataQuery metadataQuery = left.getCluster().getMetadataQuery();
  int i = 0;
  for (Integer leftKey : leftKeys) {
    String leftFieldName = leftFields.get(leftKey);
    Integer rightKey = rightKeys.get(i++);
    String rightFieldName = rightFields.get(rightKey);

    //This also avoids the left field of the join condition with a function call.
    ScanPrel scanPrel = findLeftScanPrel(leftFieldName, left);
    if (scanPrel != null) {
      boolean encounteredBlockNode = containBlockNode((Prel) left, scanPrel);
      if (encounteredBlockNode) {
        continue;
      }
      //Collect NDV from the Metadata
      RelDataType scanRowType = scanPrel.getRowType();
      RelDataTypeField field = scanRowType.getField(leftFieldName, true, true);
      int index = field.getIndex();
      Double ndv = metadataQuery.getDistinctRowCount(scanPrel, ImmutableBitSet.of(index), null);
      if (ndv == null) {
        //If NDV is not supplied, we use the row count to estimate the ndv.
        ndv = left.estimateRowCount(metadataQuery) * 0.1;
      }
      int bloomFilterSizeInBytes = BloomFilter.optimalNumOfBytes(ndv.longValue(), fpp);
      bloomFilterSizeInBytes = bloomFilterSizeInBytes > bloomFilterMaxSizeInBytesDef ? bloomFilterMaxSizeInBytesDef : bloomFilterSizeInBytes;
      //left the local parameter to be set later.
      BloomFilterDef bloomFilterDef = new BloomFilterDef(bloomFilterSizeInBytes, false, leftFieldName, rightFieldName);
      bloomFilterDef.setLeftNDV(ndv);
      bloomFilterDefs.add(bloomFilterDef);
      toAddRuntimeFilter.add(scanPrel);
      probeSideScanPrel = scanPrel;
    }
  }
  if (bloomFilterDefs.size() > 0) {
    //left sendToForeman parameter to be set later.
    RuntimeFilterDef runtimeFilterDef = new RuntimeFilterDef(true, false, bloomFilterDefs, false, -1);
    probeSideScan2hj.put(probeSideScanPrel, hashJoinPrel);
    return runtimeFilterDef;
  }
  return null;
}
 

public RelDataTypeField field(RelDataType rowType, String fieldName) {
  return rowType.getField(fieldName, caseSensitive, false);
}
 

/**
 * Creates an expression accessing a given named field from a record.
 *
 * <p>NOTE: Be careful choosing the value of {@code caseSensitive}.
 * If the field name was supplied by an end-user (e.g. as a column alias in
 * SQL), use your session's case-sensitivity setting.
 * Only hard-code {@code true} if you are sure that the field name is
 * internally generated.
 * Hard-coding {@code false} is almost certainly wrong.</p>
 *
 * @param expr      Expression yielding a record
 * @param fieldName Name of field in record
 * @param caseSensitive Whether match is case-sensitive
 * @return Expression accessing a given named field
 */
public RexNode makeFieldAccess(RexNode expr, String fieldName,
    boolean caseSensitive) {
  final RelDataType type = expr.getType();
  final RelDataTypeField field =
      type.getField(fieldName, caseSensitive, false);
  if (field == null) {
    throw new AssertionError("Type '" + type + "' has no field '"
        + fieldName + "'");
  }
  return makeFieldAccessInternal(expr, field);
}
 

/**
 * Creates an expression accessing a given named field from a record.
 *
 * <p>NOTE: Be careful choosing the value of {@code caseSensitive}.
 * If the field name was supplied by an end-user (e.g. as a column alias in
 * SQL), use your session's case-sensitivity setting.
 * Only hard-code {@code true} if you are sure that the field name is
 * internally generated.
 * Hard-coding {@code false} is almost certainly wrong.</p>
 *
 * @param expr      Expression yielding a record
 * @param fieldName Name of field in record
 * @param caseSensitive Whether match is case-sensitive
 * @return Expression accessing a given named field
 */
public RexNode makeFieldAccess(RexNode expr, String fieldName,
    boolean caseSensitive) {
  final RelDataType type = expr.getType();
  final RelDataTypeField field =
      type.getField(fieldName, caseSensitive, false);
  if (field == null) {
    throw new AssertionError("Type '" + type + "' has no field '"
        + fieldName + "'");
  }
  return makeFieldAccessInternal(expr, field);
}