com.carrotsearch.hppc.cursors.ObjectLongCursor Java Examples

The following examples show how to use com.carrotsearch.hppc.cursors.ObjectLongCursor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ReplicaShardAllocator.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up 
public MatchingNodes(ObjectLongMap<DiscoveryNode> nodesToSize) {
    this.nodesToSize = nodesToSize;

    long highestMatchSize = 0;
    DiscoveryNode highestMatchNode = null;

    for (ObjectLongCursor<DiscoveryNode> cursor : nodesToSize) {
        if (cursor.value > highestMatchSize) {
            highestMatchSize = cursor.value;
            highestMatchNode = cursor.key;
        }
    }
    this.nodeWithHighestMatch = highestMatchNode;
}
Example #2
Source File: ParquetGroupScanUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
public void setEndpointByteMap(EndpointByteMap byteMap) {
  this.byteMap = byteMap;
  this.affinities = Lists.newArrayList();
  final Iterator<ObjectLongCursor<HostAndPort>> hostPortIterator = byteMap.iterator();
  while (hostPortIterator.hasNext()) {
    ObjectLongCursor<HostAndPort> nodeEndPoint = hostPortIterator.next();
    affinities.add(EndpointAffinity.fromHostAndPort(nodeEndPoint.key, nodeEndPoint.value));
  }
}
Example #3
Source File: ReplicaShardAllocator.java    From crate with Apache License 2.0 5 votes vote down vote up 
MatchingNodes(ObjectLongMap<DiscoveryNode> nodesToSize, @Nullable Map<String, NodeAllocationResult> nodeDecisions) {
    this.nodesToSize = nodesToSize;
    this.nodeDecisions = nodeDecisions;

    long highestMatchSize = 0;
    DiscoveryNode highestMatchNode = null;

    for (ObjectLongCursor<DiscoveryNode> cursor : nodesToSize) {
        if (cursor.value > highestMatchSize) {
            highestMatchSize = cursor.value;
            highestMatchNode = cursor.key;
        }
    }
    this.nodeWithHighestMatch = highestMatchNode;
}
Example #4
Source File: EndpointByteMapImpl.java    From Bats with Apache License 2.0 4 votes vote down vote up 
@Override
public Iterator<ObjectLongCursor<DrillbitEndpoint>> iterator() {
  return map.iterator();
}
Example #5
Source File: EndpointByteMapImpl.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
@Override
public Iterator<ObjectLongCursor<HostAndPort>> iterator() {
  return map.iterator();
}
Example #6
Source File: IcebergTableWrapper.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
DatasetSplit from(FileScanTask task) throws IOException {
  // TODO ravindra: iceberg does not track counts at a row-group level. We should fallback to
  // an alternate codepath for this.
  DataFile dataFile = task.file();
  if (dataFile.splitOffsets() != null && dataFile.splitOffsets().size() > 1) {
    throw new UnsupportedOperationException("iceberg does not support multiple row groups yet");
  }
  String pathString = dataFile.path().toString();
  FileAttributes fileAttributes = fs.getFileAttributes(Path.of(pathString));

  // Get the per-column value counts. The counts may not be present.
  List<ColumnValueCount> columnValueCounts = Lists.newArrayList();
  if (dataFile.valueCounts() != null) {
    for (Map.Entry<Integer, Long> entry : dataFile.valueCounts().entrySet()) {
      String columnName = schema.findColumnName(entry.getKey());
      Types.NestedField field = schema.findField(entry.getKey());
      if (field == null || columnName  == null) {
        // we are not updating counts for list elements.
        continue;
      }

      Long totalCount = entry.getValue();
      Long nullValueCount = dataFile.nullValueCounts().get(entry.getKey());
      if (totalCount == null || nullValueCount == null) {
        continue;
      }

      long nonNullValueCount = totalCount - nullValueCount;
      columnValueCounts.add(
          ColumnValueCount.newBuilder()
              .setColumn(columnName)
              .setCount(nonNullValueCount)
              .build());
      // aggregate into the dataset level column-value counts.
      datasetColumnValueCounts.merge(
          columnName, nonNullValueCount, (x, y) -> y + nonNullValueCount);
    }
  }

  // Create the split-level xattr with details about the column counts.
  // TODO: not populating column bounds here since they are not used by dremio planner.
  ParquetDatasetSplitXAttr splitExtended =
      ParquetDatasetSplitXAttr.newBuilder()
          .setPath(pathString)
          .setStart(task.start())
          .setRowGroupIndex(0)
          .setUpdateKey(
              FileSystemCachedEntity.newBuilder()
                  .setPath(pathString)
                  .setLastModificationTime(fileAttributes.lastModifiedTime().toMillis())
                  .setLength(fileAttributes.size()))
          .addAllColumnValueCounts(columnValueCounts)
          .setLength(task.length())
          .build();

  // build the host affinity details for the split.
  Map<HostAndPort, Float> affinities =
    Metadata.getHostAffinity(fs, fileAttributes, task.start(), task.length());
  List<DatasetSplitAffinity> splitAffinities = new ArrayList<>();
  for (ObjectLongCursor<HostAndPort> item :
    ParquetGroupScanUtils.buildEndpointByteMap(activeHostMap,
      activeHostPortMap, affinities, task.length())) {
    splitAffinities.add(DatasetSplitAffinity.of(item.key.toString(), item.value));
  }

  return DatasetSplit.of(
      splitAffinities, task.length(), task.file().recordCount(), splitExtended::writeTo);
}