com.carrotsearch.hppc.cursors.IntCursor Java Examples

@Override
public void writeTo(StreamOutput out) throws IOException {
    out.writeVInt(locations.size());
    for (Map.Entry<String, Map<String, IntIndexedContainer>> entry : locations.entrySet()) {
        out.writeString(entry.getKey());

        Map<String, IntIndexedContainer> shardsByIndex = entry.getValue();
        if (shardsByIndex == null) {
            out.writeVInt(0);
        } else {
            out.writeVInt(shardsByIndex.size());
            for (Map.Entry<String, IntIndexedContainer> innerEntry : shardsByIndex.entrySet()) {
                out.writeString(innerEntry.getKey());
                IntIndexedContainer shardIds = innerEntry.getValue();
                if (shardIds == null || shardIds.size() == 0) {
                    out.writeVInt(0);
                } else {
                    out.writeVInt(shardIds.size());
                    for (IntCursor shardId: shardIds) {
                        out.writeVInt(shardId.value);
                    }
                }
            }
        }
    }
}
 

@Test
public void shouldIterateOverIndexesAndSymbolsNoBacking() throws Exception {
  MutableSymbolTable base = new MutableSymbolTable();
  int a = base.getOrAdd("A");
  int b = base.getOrAdd("B");
  UnionSymbolTable union = new UnionSymbolTable(base);

  Set<String> valuesFromIterable = Sets.newHashSet();
  for (String symbol : union.symbols()) {
    assertTrue(valuesFromIterable.add(symbol));
  }
  assertEquals(ImmutableSet.of("A", "B"), valuesFromIterable);

  Set<Integer> indexesFromIterable = Sets.newHashSet();
  for (IntCursor cursor : union.indexes()) {
    assertTrue(indexesFromIterable.add(cursor.value));
  }
  assertEquals(ImmutableSet.of(a, b), indexesFromIterable);
}
 

@Test
public void shouldIterateOverIndexesAndSymbolsWithBacking() throws Exception {
  MutableSymbolTable base = new MutableSymbolTable();
  int a = base.getOrAdd("A");
  int b = base.getOrAdd("B");
  UnionSymbolTable union = new UnionSymbolTable(base);
  int c = union.getOrAdd("C");

  Set<String> valuesFromIterable = Sets.newHashSet();
  for (String symbol : union.symbols()) {
    assertTrue(valuesFromIterable.add(symbol));
  }
  assertEquals(ImmutableSet.of("A", "B", "C"), valuesFromIterable);

  Set<Integer> indexesFromIterable = Sets.newHashSet();
  for (IntCursor cursor : union.indexes()) {
    assertTrue(indexesFromIterable.add(cursor.value));
  }
  assertEquals(ImmutableSet.of(a, b, c), indexesFromIterable);
}
 

@Test
public void shouldSupportPutWithId() throws Exception {
  MutableSymbolTable base = new MutableSymbolTable();
  int a = base.getOrAdd("A");
  UnionSymbolTable union = new UnionSymbolTable(base);
  int b = union.getOrAdd("B");
  union.put("C", 42);
  int d = union.getOrAdd("D");

  Set<String> valuesFromIterable = Sets.newHashSet();
  for (String symbol : union.symbols()) {
    assertTrue(valuesFromIterable.add(symbol));
  }
  assertEquals(ImmutableSet.of("A", "B", "C", "D"), valuesFromIterable);

  Set<Integer> indexesFromIterable = Sets.newHashSet();
  for (IntCursor cursor : union.indexes()) {
    assertTrue(indexesFromIterable.add(cursor.value));
  }
  assertEquals(ImmutableSet.of(a, b, 42, d), indexesFromIterable);
}
 

/**
 * Serialize the list of terms to the {@link StreamOutput}.
 * <br>
 * Given the low performance of {@link org.elasticsearch.common.io.stream.BytesStreamOutput} when writing a large number
 * of longs (5 to 10 times slower than writing directly to a byte[]), we use a small buffer of 8kb
 * to optimise the throughput. 8kb seems to be the optimal buffer size, larger buffer size did not improve
 * the throughput.
 *
 * @param out the output
 */
@Override
public void writeTo(StreamOutput out) throws IOException {
  // Encode flag
  out.writeBoolean(this.isPruned());

  // Encode size of list
  out.writeInt(set.size());

  // Encode ints
  BytesRef buffer = new BytesRef(new byte[1024 * 8]);
  Iterator<IntCursor> it = set.iterator();
  while (it.hasNext()) {
    Bytes.writeVInt(buffer, it.next().value);
    if (buffer.offset > buffer.bytes.length - 5) {
      out.write(buffer.bytes, 0, buffer.offset);
      buffer.offset = 0;
    }
  }

  // flush the remaining bytes from the buffer
  out.write(buffer.bytes, 0, buffer.offset);
}
 

protected static int[] convertToInt(IntArrayList[] setToSplit, int exclude) {
  int count = 0;

  for (int i = 0; i < setToSplit.length; i++) {
    if (i != exclude) {
      count += setToSplit[i].size();
    }
  }

  int[] setData = new int[count];
  int a = 0;
  for (int i = 0; i < setToSplit.length; i++) {
    if (i != exclude) {
      for (IntCursor d : setToSplit[i]) {
        setData[a++] = d.value;
      }
    }
  }

  return setData;
}
 

/**
 * Returns a direct iterator over the keys.
 */
public Iterator<Integer> keysIt() {
    final Iterator<IntCursor> iterator = map.keys().iterator();
    return new Iterator<Integer>() {
        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public Integer next() {
            return iterator.next().value;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    };
}
 

/**
 * Converts the shardId's of each node->table from a {@link IntIndexedContainer} to a list of Integers as custom
 * classes are not supported by the {@link org.elasticsearch.common.xcontent.XContentBuilder}.
 */
private static Map<String, Map<String, List<Integer>>> xContentSafeRoutingLocations(
    Map<String, Map<String, IntIndexedContainer>> locations) {
    HashMap<String, Map<String, List<Integer>>> safeLocations = new HashMap<>(locations.size(), 1f);
    for (Map.Entry<String, Map<String, IntIndexedContainer>> nodeEntry : locations.entrySet()) {
        HashMap<String, List<Integer>> tableShards = new HashMap<>(nodeEntry.getValue().size(), 1f);
        for (Map.Entry<String, IntIndexedContainer> tableEntry : nodeEntry.getValue().entrySet()) {
            ArrayList<Integer> shardList = new ArrayList<>(tableEntry.getValue().size());
            for (IntCursor cursor : tableEntry.getValue()) {
                shardList.add(cursor.value);
            }
            // ensure a deterministic shard list by sorting it (important for test assertions but maybe also for apps)
            shardList.sort(Integer::compareTo);
            tableShards.put(tableEntry.getKey(), shardList);
        }
        safeLocations.put(nodeEntry.getKey(), tableShards);
    }
    return safeLocations;
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeLong(jobId.getMostSignificantBits());
    out.writeLong(jobId.getLeastSignificantBits());
    out.writeVInt(fetchPhaseId);
    if (toFetch == null) {
        out.writeVInt(0);
    } else {
        out.writeVInt(toFetch.size());
        for (IntObjectCursor<IntContainer> toFetchCursor : toFetch) {
            out.writeVInt(toFetchCursor.key);
            out.writeVInt(toFetchCursor.value.size());
            for (IntCursor docCursor : toFetchCursor.value) {
                out.writeInt(docCursor.value);
            }
        }
    }
}
 

@SuppressWarnings({"rawtypes"})
public Map<String, ? extends IntObjectMap<Streamer[]>> generateStreamersGroupedByReaderAndNode() {
    HashMap<String, IntObjectHashMap<Streamer[]>> streamersByReaderByNode = new HashMap<>();
    for (Map.Entry<String, IntSet> entry : nodeReaders.entrySet()) {
        IntObjectHashMap<Streamer[]> streamersByReaderId = new IntObjectHashMap<>();
        String nodeId = entry.getKey();
        streamersByReaderByNode.put(nodeId, streamersByReaderId);
        for (IntCursor readerIdCursor : entry.getValue()) {
            int readerId = readerIdCursor.value;
            String index = readerIndices.floorEntry(readerId).getValue();
            RelationName relationName = indicesToIdents.get(index);
            FetchSource fetchSource = fetchSources.get(relationName);
            if (fetchSource == null) {
                continue;
            }
            streamersByReaderId.put(readerIdCursor.value, Symbols.streamerArray(fetchSource.references()));
        }
    }
    return streamersByReaderByNode;
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeLong(jobId.getMostSignificantBits());
    out.writeLong(jobId.getLeastSignificantBits());
    out.writeVInt(fetchPhaseId);
    out.writeBoolean(closeContext);
    if (toFetch == null) {
        out.writeVInt(0);
    } else {
        out.writeVInt(toFetch.size());
        for (IntObjectCursor<? extends IntContainer> toFetchCursor : toFetch) {
            out.writeVInt(toFetchCursor.key);
            out.writeVInt(toFetchCursor.value.size());
            for (IntCursor docCursor : toFetchCursor.value) {
                out.writeInt(docCursor.value);
            }
        }
    }
}
 

@Override
public Iterable<IntCursor> indexes() {
  if (filter == null) {
    return backing.indexes();
  }
  return Iterables.concat(backing.indexes(), filter.indexes());
}
 

public Builder addReplica() {
    for (IntCursor cursor : shards.keys()) {
        int shardNumber = cursor.value;
        ShardId shardId = new ShardId(index, shardNumber);
        // version 0, will get updated when reroute will happen
        ShardRouting shard = ShardRouting.newUnassigned(shardId, false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null));
        shards.put(shardNumber,
                new IndexShardRoutingTable.Builder(shards.get(shard.id())).addShard(shard).build()
        );
    }
    return this;
}
 

public StreamBucket collect(IntContainer docIds) {
    StreamBucket.Builder builder = new StreamBucket.Builder(streamers, ramAccounting);
    for (IntCursor cursor : docIds) {
        int docId = cursor.value;
        int readerIndex = ReaderUtil.subIndex(docId, readerContexts);
        LeafReaderContext subReaderContext = readerContexts.get(readerIndex);
        try {
            setNextDocId(subReaderContext, docId - subReaderContext.docBase);
        } catch (IOException e) {
            Exceptions.rethrowRuntimeException(e);
        }
        builder.add(row);
    }
    return builder.build();
}
 

private static int getMax(IntIndexedContainer shards) {
    int max = -1;
    for (IntCursor shard: shards) {
        if (shard.value > max) {
            max = shard.value;
        }
    }
    return max;
}
 

@Override
public void write(Kryo kryo, Output output, BulkWriteResult object) {
		kryo.writeObject(output,object.globalStatus);
		output.writeInt(object.notRunRows.size());
		for(IntCursor cursor:object.notRunRows){
				output.writeInt(cursor.value);
		}
		output.writeInt(object.failedRows.size());
		for(IntObjectCursor<WriteResult> c:object.failedRows){
				output.writeInt(c.key);
				kryo.writeObject(output,c.value);
		}
}
 

public IntObjectHashMap<IntContainer> generateToFetch(IntSet readerIds) {
    IntObjectHashMap<IntContainer> toFetch = new IntObjectHashMap<>(readerIds.size());
    for (IntCursor readerIdCursor : readerIds) {
        ReaderBucket readerBucket = readerBuckets.get(readerIdCursor.value);
        if (readerBucket != null && readerBucket.docs.size() > 0) {
            toFetch.put(readerIdCursor.value, readerBucket.docs.keys());
        }
    }
    return toFetch;
}
 

public boolean upstreamsAreOnSameNode(int phaseId) {
    IntContainer sourcePhases = targetToSourceMap.get(phaseId);
    if (sourcePhases == null) {
        return false;
    }
    for (IntCursor sourcePhase : sourcePhases) {
        NodeOperation nodeOperation = nodeOperationByPhaseId.get(sourcePhase.value);
        if (nodeOperation == null) {
            return false;
        }
        ExecutionPhase executionPhase = nodeOperation.executionPhase();
        // explicit SAME_NODE distribution enforced by the planner
        if (executionPhase instanceof UpstreamPhase &&
            ((UpstreamPhase) executionPhase).distributionInfo().distributionType() ==
            DistributionType.SAME_NODE) {
            continue;
        }

        // implicit same node optimization because the upstreamPhase is running ONLY on this node
        Collection<String> executionNodes = executionPhase.nodeIds();
        if (executionNodes.size() == 1 && executionNodes.iterator().next().equals(localNodeId)) {
            continue;
        }
        return false;
    }
    return true;
}
 

@Override
public BytesRef writeToBytes() {
  long start = System.nanoTime();
  int size = set.size();

  BytesRef bytesRef = new BytesRef(new byte[HEADER_SIZE + size * 5]);

  // Encode encoding type
  Bytes.writeInt(bytesRef, this.getEncoding().ordinal());

  // Encode flag
  bytesRef.bytes[bytesRef.offset++] = (byte) (this.isPruned() ? 1 : 0);

  // Encode size of list
  Bytes.writeInt(bytesRef, size);

  // Encode ints
  for (IntCursor i : set) {
    Bytes.writeVInt(bytesRef, i.value);
  }

  logger.debug("Serialized {} terms - took {} ms", this.size(), (System.nanoTime() - start) / 1000000);

  bytesRef.length = bytesRef.offset;
  bytesRef.offset = 0;
  return bytesRef;
}
 

public int[] getAll() {
    KeysContainer keys = keys();
    int[] all = new int[keys.size()];
    Iterator<IntCursor> iter = keys.iterator();
    int pos=0;
    while (iter.hasNext()) all[pos++]=iter.next().value;
    return all;
}
 

protected static int[] convertToInt(IntArrayList trainSet) {
  int[] train = new int[trainSet.size()];
  int a = 0;
  for (IntCursor i : trainSet) {
    train[a++] = i.value;
  }
  return train;
}
 

/**
 * Approximate search for the query.
 */
public SortedListMap<Double, Integer> search(byte[] wordQuery, TimeSeries query, int k) {
  SortedListMap<Double, Integer> result = new SortedListMap<>(k);

  // search for the exact path
  SFANode node = getLeafNode(wordQuery);

  // leaf node
  if (node != null && node.type == NodeType.Leaf) {
    addToIOTimeSeriesRead(1);
    addToTimeSeriesRead(node.getSize());

    // retrieve all time series
    for (IntCursor idx : node.getElementIds()) {
      double distance = getEuclideanDistance(
          type == MatchingType.Subsequences ? timeSeries[0] : timeSeries[idx.value],
          query,
          means[idx.value],
          stddev[idx.value],
          Double.MAX_VALUE,
          type == MatchingType.Subsequences ? idx.value : 0);
      result.put(distance, idx.value);
    }
    return result;
  } else {
    throw new RuntimeException("No path found!");
  }
}
 

public void forEachValue(IntConsumer consumer) {
  if (keyValues != null) {
    for (int val : keyValues) {
      if (val != emptyValue) consumer.accept(val);
    }
  } else {
    for (IntCursor ord : hashMap.values()) {
      consumer.accept(ord.value);
    }
  }
}
 

/**
 * Returns a direct iterator over the keys.
 */
public UnmodifiableIterator<Integer> keysIt() {
    final Iterator<IntCursor> iterator = map.keys().iterator();
    return new UnmodifiableIterator<Integer>() {
        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public Integer next() {
            return iterator.next().value;
        }
    };
}
 

public Builder addReplica() {
    for (IntCursor cursor : shards.keys()) {
        int shardId = cursor.value;
        // version 0, will get updated when reroute will happen
        ShardRouting shard = ShardRouting.newUnassigned(index, shardId, null, false, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null));
        shards.put(shardId,
                new IndexShardRoutingTable.Builder(shards.get(shard.id())).addShard(shard).build()
        );
    }
    return this;
}
 

private static void convertStringColumns(Object[] row, IntArrayList stringColumns) {
    for (IntCursor stringColumn : stringColumns) {
        Object value = row[stringColumn.value];
        if (value instanceof BytesRef) {
            row[stringColumn.value] = ((BytesRef)value).utf8ToString();
        }
    }
}
 

private static void convertStringCollectionColumns(Object[] row, IntArrayList stringCollectionColumns) {
    for (IntCursor stringCollectionColumn : stringCollectionColumns) {
        Object value = row[stringCollectionColumn.value];
        if (value == null) {
            continue;
        }
        if (value instanceof Set) {
            row[stringCollectionColumn.value] = setToStringArray(((Set<BytesRef>) value));
        } else if (value instanceof BytesRef[]) {
            row[stringCollectionColumn.value] = objectArrayToStringArray(((BytesRef[]) value));
        } else if (value instanceof Object[]) {
            row[stringCollectionColumn.value] = objectArrayToStringArray(((Object[]) value));
        }
    }
}
 

public void collect(IntContainer docIds, StreamBucket.Builder builder) throws IOException {
    for (IntCursor cursor : docIds) {
        final int docId = cursor.value;
        int readerIndex = ReaderUtil.subIndex(docId, readerContexts);
        LeafReaderContext subReaderContext = readerContexts.get(readerIndex);
        setNextReader(subReaderContext);
        setNextDocId(docId - subReaderContext.docBase);
        builder.add(row);
    }
}
 

/**
 * find all phases that don't have any downstreams.
 * <p>
 * This is usually only one phase (the handlerPhase, but there might be more in case of bulk operations)
 * <pre>
 *
 *     flow:            targetToSourceMap:
 *
 *     0    1               2 -> [0, 1]
 *      \  /                3 -> [2]
 *       2
 *       |
 *       3
 *
 *     leafs = all keys in targetToSource map which have no entry in values (3 in the example above)
 * </pre>
 */
private static IntCollection findLeafs(IntObjectMap<? extends IntContainer> targetToSourceMap) {
    IntArrayList leafs = new IntArrayList();
    BitSet sources = new BitSet();
    for (IntObjectCursor<? extends IntContainer> sourceIds : targetToSourceMap) {
        for (IntCursor sourceId : sourceIds.value) {
            sources.set(sourceId.value);
        }
    }
    for (IntCursor targetPhaseCursor : targetToSourceMap.keys()) {
        int targetPhase = targetPhaseCursor.value;
        if (!sources.get(targetPhase)) {
            leafs.add(targetPhase);
        }
    }
    return leafs;
}
 

public Iterable<? extends IntCursor> findLeafs() {
    return findLeafs(targetToSourceMap);
}