Java Code Examples for org.apache.calcite.rel.type.RelDataTypeField#getIndex()

/**
 * Creates an expression accessing a given field from a record.
 *
 * @param expr  Expression yielding a record
 * @param field Field
 * @return Expression accessing given field
 */
private RexNode makeFieldAccessInternal(
    RexNode expr,
    final RelDataTypeField field) {
  if (expr instanceof RexRangeRef) {
    RexRangeRef range = (RexRangeRef) expr;
    if (field.getIndex() < 0) {
      return makeCall(
          field.getType(),
          GET_OPERATOR,
          ImmutableList.of(
              expr,
              makeLiteral(field.getName())));
    }
    return rexFactory.makeInputRef(
        range.getOffset() + field.getIndex(),
        field.getType());
  }
  return rexFactory.makeFieldAccess(expr, field);
}
 

public RelDataType constructRowType(RelDataType inputRowType) {
  Preconditions.checkArgument(this.requiredColumns.cardinality() == 1);

  List<RelDataType> fields = new ArrayList<>();
  List<String> fieldNames = new ArrayList<>();
  if (excludeCorrelateColumn) {
    int corrVariable = this.requiredColumns.nextSetBit(0);

    for (RelDataTypeField field : inputRowType.getFieldList()) {
      if (field.getIndex() == corrVariable) {
        continue;
      }
      fieldNames.add(field.getName());
      fields.add(field.getType());
    }

    return getCluster().getTypeFactory().createStructType(fields, fieldNames);
  }
  return inputRowType;
}
 

/**
 * Creates an expression accessing a given field from a record.
 *
 * @param expr  Expression yielding a record
 * @param field Field
 * @return Expression accessing given field
 */
private RexNode makeFieldAccessInternal(
    RexNode expr,
    final RelDataTypeField field) {
  if (expr instanceof RexRangeRef) {
    RexRangeRef range = (RexRangeRef) expr;
    if (field.getIndex() < 0) {
      return makeCall(
          field.getType(),
          GET_OPERATOR,
          ImmutableList.of(
              expr,
              makeLiteral(field.getName())));
    }
    return new RexInputRef(
        range.getOffset() + field.getIndex(),
        field.getType());
  }
  return new RexFieldAccess(expr, field);
}
 

private String getString(RelDataType rowType) {
  StringBuilder buf = new StringBuilder();
  buf.append("(");
  for (RelDataTypeField field : rowType.getFieldList()) {
    if (field.getIndex() > 0) {
      buf.append(", ");
    }
    buf.append("'");
    buf.append(field.getName());
    buf.append("'");
  }
  buf.append(")");
  return buf.toString();
}
 

private String getString(RelDataType rowType) {
  StringBuilder buf = new StringBuilder();
  buf.append("(");
  for (RelDataTypeField field : rowType.getFieldList()) {
    if (field.getIndex() > 0) {
      buf.append(", ");
    }
    buf.append("'");
    buf.append(field.getName());
    buf.append("'");
  }
  buf.append(")");
  return buf.toString();
}
 

/**
 * Generate a possible RuntimeFilter of a HashJoinPrel, left some BF parameters of the generated RuntimeFilter
 * to be set later.
 *
 * @param hashJoinPrel
 * @return null or a partial information RuntimeFilterDef
 */
private RuntimeFilterDef generateRuntimeFilter(HashJoinPrel hashJoinPrel) {
  JoinRelType joinRelType = hashJoinPrel.getJoinType();
  JoinInfo joinInfo = hashJoinPrel.analyzeCondition();
  boolean allowJoin = (joinInfo.isEqui()) && (joinRelType == JoinRelType.INNER || joinRelType == JoinRelType.RIGHT);
  if (!allowJoin) {
    return null;
  }
  //TODO check whether to enable RuntimeFilter according to the NDV percent
  /**
   double threshold = 0.5;
   double percent = leftNDV / rightDNV;
   if (percent > threshold ) {
   return null;
   }
   */

  List<BloomFilterDef> bloomFilterDefs = new ArrayList<>();
  //find the possible left scan node of the left join key
  ScanPrel probeSideScanPrel = null;
  RelNode left = hashJoinPrel.getLeft();
  RelNode right = hashJoinPrel.getRight();
  ExchangePrel exchangePrel = findRightExchangePrel(right);
  if (exchangePrel == null) {
    //Does not support the single fragment mode ,that is the right build side
    //can only be BroadcastExchangePrel or HashToRandomExchangePrel
    return null;
  }
  List<String> leftFields = left.getRowType().getFieldNames();
  List<String> rightFields = right.getRowType().getFieldNames();
  List<Integer> leftKeys = hashJoinPrel.getLeftKeys();
  List<Integer> rightKeys = hashJoinPrel.getRightKeys();
  RelMetadataQuery metadataQuery = left.getCluster().getMetadataQuery();
  int i = 0;
  for (Integer leftKey : leftKeys) {
    String leftFieldName = leftFields.get(leftKey);
    Integer rightKey = rightKeys.get(i++);
    String rightFieldName = rightFields.get(rightKey);

    //This also avoids the left field of the join condition with a function call.
    ScanPrel scanPrel = findLeftScanPrel(leftFieldName, left);
    if (scanPrel != null) {
      boolean encounteredBlockNode = containBlockNode((Prel) left, scanPrel);
      if (encounteredBlockNode) {
        continue;
      }
      //Collect NDV from the Metadata
      RelDataType scanRowType = scanPrel.getRowType();
      RelDataTypeField field = scanRowType.getField(leftFieldName, true, true);
      int index = field.getIndex();
      Double ndv = metadataQuery.getDistinctRowCount(scanPrel, ImmutableBitSet.of(index), null);
      if (ndv == null) {
        //If NDV is not supplied, we use the row count to estimate the ndv.
        ndv = left.estimateRowCount(metadataQuery) * 0.1;
      }
      int bloomFilterSizeInBytes = BloomFilter.optimalNumOfBytes(ndv.longValue(), fpp);
      bloomFilterSizeInBytes = bloomFilterSizeInBytes > bloomFilterMaxSizeInBytesDef ? bloomFilterMaxSizeInBytesDef : bloomFilterSizeInBytes;
      //left the local parameter to be set later.
      BloomFilterDef bloomFilterDef = new BloomFilterDef(bloomFilterSizeInBytes, false, leftFieldName, rightFieldName);
      bloomFilterDef.setLeftNDV(ndv);
      bloomFilterDefs.add(bloomFilterDef);
      toAddRuntimeFilter.add(scanPrel);
      probeSideScanPrel = scanPrel;
    }
  }
  if (bloomFilterDefs.size() > 0) {
    //left sendToForeman parameter to be set later.
    RuntimeFilterDef runtimeFilterDef = new RuntimeFilterDef(true, false, bloomFilterDefs, false, -1);
    probeSideScan2hj.put(probeSideScanPrel, hashJoinPrel);
    return runtimeFilterDef;
  }
  return null;
}
 

private RelDataTypeField dictionaryEncodedField(RelDataTypeField field) {
  return new RelDataTypeFieldImpl(field.getName(), field.getIndex(), dictionaryDataType);
}
 

private PrelWithDictionaryInfo visitParquetScanPrel(ParquetScanPrel parquetScanPrel, Void value) throws RuntimeException {
  final ReadDefinition readDefinition = parquetScanPrel.getTableMetadata().getReadDefinition();

  if (readDefinition == null || readDefinition.getExtendedProperty() == null) {
    return new PrelWithDictionaryInfo(parquetScanPrel);
  }

  // Make sure we don't apply global dictionary on columns that have conditions pushed into the scan
  final Set<String> columnsPushedToScan = new HashSet<>();
  if (parquetScanPrel.getFilter() != null) {
    Iterables.addAll(columnsPushedToScan,
        Iterables.transform(parquetScanPrel.getFilter().getConditions(), ParquetFilterCondition.EXTRACT_COLUMN_NAME));
  }

  final Map<String, String> dictionaryEncodedColumnsToDictionaryFilePath = Maps.newHashMap();
  long dictionaryVersion = -1;

  ParquetDatasetXAttr xAttr = null;

  boolean isIcebergDataset = isIcebergDataset(parquetScanPrel.getTableMetadata());
  // Dremio 5.0 release had ParquetDatasetXAttr in extended property
  boolean newIcebergDataset = false;
  if (isIcebergDataset) {
    try {
      IcebergDatasetXAttr icebergDatasetXAttr = LegacyProtobufSerializer.parseFrom(IcebergDatasetXAttr.PARSER,
        readDefinition.getExtendedProperty().asReadOnlyByteBuffer());
      xAttr = icebergDatasetXAttr.getParquetDatasetXAttr();
      newIcebergDataset = true;
    } catch (InvalidProtocolBufferException ignored) {
    }
  }

  if (!isIcebergDataset || !newIcebergDataset) {
    try {
      xAttr = LegacyProtobufSerializer.parseFrom(ParquetDatasetXAttr.PARSER,
        readDefinition.getExtendedProperty().asReadOnlyByteBuffer());
    } catch (InvalidProtocolBufferException e) {
      if (isIcebergDataset) {
        throw new RuntimeException("Could not deserialize Iceberg dataset info");
      } else {
        throw new RuntimeException("Could not deserialize parquet dataset info");
      }
    }
  }

  if (xAttr.hasDictionaryEncodedColumns()) {
    final DictionaryEncodedColumns dictionaryEncodedColumns = xAttr.getDictionaryEncodedColumns();
    dictionaryVersion = dictionaryEncodedColumns.getVersion();
    // Construct paths to dictionary files based on the version found in namespace. Do NOT look for files during planning.
    final Path dictionaryRootPath = Path.of(dictionaryEncodedColumns.getRootPath());
    for (String dictionaryEncodedColumn : dictionaryEncodedColumns.getColumnsList()) {
      if (!columnsPushedToScan.contains(dictionaryEncodedColumn)) {
        dictionaryEncodedColumnsToDictionaryFilePath.put(dictionaryEncodedColumn,
          GlobalDictionaryBuilder.dictionaryFilePath(dictionaryRootPath, dictionaryEncodedColumn).toString());
      }
    }
  }

  if (dictionaryEncodedColumnsToDictionaryFilePath.isEmpty()) {
    return new PrelWithDictionaryInfo(parquetScanPrel);
  }

  final StoragePluginId storagePluginId = parquetScanPrel.getPluginId();
  boolean encodedColumns = false;
  final List<RelDataTypeField> newFields = Lists.newArrayList();
  final GlobalDictionaryFieldInfo[] fieldInfos = new GlobalDictionaryFieldInfo[parquetScanPrel.getRowType().getFieldCount()];
  final List<GlobalDictionaryFieldInfo> globalDictionaryColumns = Lists.newArrayList();

  for (int i = 0; i < parquetScanPrel.getRowType().getFieldCount(); ++i) {
    final RelDataTypeField field = parquetScanPrel.getRowType().getFieldList().get(i);
    if (dictionaryEncodedColumnsToDictionaryFilePath.containsKey(field.getName())) {
      fieldInfos[i] = new GlobalDictionaryFieldInfo(
        dictionaryVersion,
        field.getName(),
        storagePluginId,
        CalciteArrowHelper.fromRelAndMinorType(field
          .getType(), TypeInferenceUtils.getMinorTypeFromCalciteType(field
          .getType())).getType(),
        dictionaryEncodedColumnsToDictionaryFilePath.get(field.getName()),
        new RelDataTypeFieldImpl(field.getName(), field.getIndex(), field.getType()));
      newFields.add(dictionaryEncodedField(field));
      globalDictionaryColumns.add(fieldInfos[i]);
      encodedColumns = true;
    } else {
      fieldInfos[i] = null;
      newFields.add(field);
    }
  }

  if (!encodedColumns) {
    return new PrelWithDictionaryInfo(parquetScanPrel);
  }

  final RelDataType newRelDataType = PrelWithDictionaryInfo.toRowDataType(newFields, parquetScanPrel.getCluster().getTypeFactory());
  final ParquetScanPrel newParquetScanPrel = parquetScanPrel.cloneWithGlobalDictionaryColumns(globalDictionaryColumns, newRelDataType);
  return new PrelWithDictionaryInfo(newParquetScanPrel, fieldInfos);
}