Java Code Examples for com.carrotsearch.hppc.IntArrayList#add()

/**
 * Factorize a gallic weight into the (head x weight, rest x One); the contract of factorize is that:
 * val (factor1, factor2) = factorize(weight) implies weight = times(factor1, factor2)
 * (see openfst's GallicFactor)
 * @param weight gallic weight to factorize
 * @return
 */
public Pair<GallicWeight, GallicWeight> factorize(GallicWeight weight) {
  Preconditions.checkArgument(isNotZero(weight), "cannot factorize a zero weight");
  IntArrayList labels = weight.getLabels();
  if (labels.isEmpty()) {
    return Pair.of(GallicWeight.createEmptyLabels(weight.getWeight()), one());
  }
  if (labels.size() == 1) {
    return Pair.of(GallicWeight.createSingleLabel(labels.get(0), weight.getWeight()), one());
  }
  IntArrayList prefix = new IntArrayList(1);
  IntArrayList suffix = new IntArrayList(labels.size() - 1);
  prefix.add(labels.get(0));
  for (int i = 1; i < labels.size(); i++) {
    suffix.add(labels.get(i));
  }
  return Pair.of(GallicWeight.create(prefix, weight.getWeight()), GallicWeight.create(suffix, weightSemiring.one()));
}
 

public NodeFetchRequest(StreamInput in) throws IOException {
    super(in);
    jobId = new UUID(in.readLong(), in.readLong());
    fetchPhaseId = in.readVInt();
    closeContext = in.readBoolean();
    int numReaders = in.readVInt();
    if (numReaders > 0) {
        IntObjectHashMap<IntArrayList> toFetch = new IntObjectHashMap<>(numReaders);
        for (int i = 0; i < numReaders; i++) {
            int readerId = in.readVInt();
            int numDocs = in.readVInt();
            IntArrayList docs = new IntArrayList(numDocs);
            toFetch.put(readerId, docs);
            for (int j = 0; j < numDocs; j++) {
                docs.add(in.readInt());
            }
        }
        this.toFetch = toFetch;
    } else {
        this.toFetch = null;
    }
}
 

public Routing(StreamInput in) throws IOException {
    int numLocations = in.readVInt();
    if (numLocations == 0) {
        locations = Map.of();
    } else {
        locations = new TreeMap<>();

        for (int i = 0; i < numLocations; i++) {
            String nodeId = in.readString();
            int numInner = in.readVInt();
            Map<String, IntIndexedContainer> shardsByIndex = new TreeMap<>();

            locations.put(nodeId, shardsByIndex);
            for (int j = 0; j < numInner; j++) {
                String indexName = in.readString();
                int numShards = in.readVInt();
                IntArrayList shardIds = new IntArrayList(numShards);
                for (int k = 0; k < numShards; k++) {
                    shardIds.add(in.readVInt());
                }
                shardsByIndex.put(indexName, shardIds);
            }
        }
    }
}
 

/**
 * Returns all transformations that do not have the given property and sorts the resulting array
 * according to the strategy.
 *
 * @param level The level which is to be sorted
 * @param triggerSkip The trigger to be used for limiting the number of nodes to be sorted
 * @return A sorted array of nodes remaining on this level
 */
private int[] getSortedUnprocessedNodes(int level, DependentAction triggerSkip) {

    // Create
    IntArrayList list = new IntArrayList();
    for (ObjectIterator<Long> iter = ((SolutionSpaceLong)solutionSpace).unsafeGetLevel(level); iter.hasNext();) {
        long id = iter.next();
        if (!skip(triggerSkip, ((SolutionSpaceLong)solutionSpace).getTransformation(id))) {
            list.add((int)id);
        }            
    }

    // Copy & sort
    int[] array = new int[list.size()];
    System.arraycopy(list.buffer, 0, array, 0, list.elementsCount);
    sort(array);
    return array;
}
 

@Override
public void getDocumentsWithWord(String word, IntArrayList documents) {
    DocsEnum docs = null;
    Term term = new Term(fieldName, word);
    try {
        int baseDocId;
        for (int i = 0; i < reader.length; i++) {
            docs = reader[i].termDocsEnum(term);
            baseDocId = contexts[i].docBase;
            if (docs != null) {
                while (docs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
                    documents.add(docs.docID() + baseDocId);
                }
            }
        }
    } catch (IOException e) {
        LOGGER.error("Error while requesting documents for word \"" + word + "\".", e);
    }
}
 

public List<CompletableFuture<Long>> executeBulk(List<Row> bulkParams, SubQueryResults subQueryResults) {
    HashMap<ShardId, Req> requests = new HashMap<>();
    IntArrayList bulkIndices = new IntArrayList(bulkParams.size() * docKeys.size());
    int location = 0;
    for (int resultIdx = 0; resultIdx < bulkParams.size(); resultIdx++) {
        int prevLocation = location;
        Row params = bulkParams.get(resultIdx);
        grouper.bind(params, subQueryResults);
        location = addRequests(location, params, requests, subQueryResults);
        for (int i = prevLocation; i < location; i++) {
            bulkIndices.add(resultIdx);
        }
    }
    BulkShardResponseListener listener =
        new BulkShardResponseListener(requests.size(), bulkParams.size(), bulkIndices);
    for (Req req : requests.values()) {
        transportAction.accept(req, listener);
    }
    return listener.rowCountFutures();
}
 

public static void ensureStringTypesAreStrings(DataType[] dataTypes, Object[][] rows) {
    if (rows.length == 0) {
        return;
    }

    // NOTE: currently BytesRef inside Maps aren't converted here because
    // if the map is coming from a ESSearchTask/EsGetTask they already contain strings
    // and we have no case in which another Task returns a Map with ByteRefs/Strings inside.
    final IntArrayList stringColumns = new IntArrayList();
    final IntArrayList stringCollectionColumns = new IntArrayList();
    int idx = 0;
    for (DataType dataType : dataTypes) {
        if (BYTES_REF_TYPES.contains(dataType)) {
            stringColumns.add(idx);
        } else if ((DataTypes.isCollectionType(dataType)
                && (BYTES_REF_TYPES.contains(((CollectionType)dataType).innerType())))) {
            stringCollectionColumns.add(idx);
        }
        idx++;
    }

    for (Object[] row : rows) {
        convertStringColumns(row, stringColumns);
        convertStringCollectionColumns(row, stringCollectionColumns);
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    responses = new ArrayList<>(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            TermVectorsResponse response = new TermVectorsResponse();
            response.readFrom(in);
            responses.add(response);
        } else {
            responses.add(null);
        }
        if (in.readBoolean()) {
            failures.add(MultiTermVectorsResponse.Failure.readFailure(in));
        } else {
            failures.add(null);
        }
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    items = new ArrayList<>(size);

    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        items.add(MultiGetRequest.Item.readItem(in));
    }

    preference = in.readOptionalString();
    refresh = in.readBoolean();
    byte realtime = in.readByte();
    if (realtime == 0) {
        this.realtime = false;
    } else if (realtime == 1) {
        this.realtime = true;
    }
    ignoreErrorsOnGeneratedFields = in.readBoolean();
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    responses = new ArrayList<>(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            GetResponse response = new GetResponse();
            response.readFrom(in);
            responses.add(response);
        } else {
            responses.add(null);
        }
        if (in.readBoolean()) {
            failures.add(MultiGetResponse.Failure.readFailure(in));
        } else {
            failures.add(null);
        }
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            failures.add(Failure.readFailure(in));
        } else {
            failures.add(null);
        }
    }
    if (in.readBoolean()) {
        failure = in.readThrowable();
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    jobId = new UUID(in.readLong(), in.readLong());
    fetchPhaseId = in.readVInt();
    int numReaders = in.readVInt();
    if (numReaders > 0) {
        toFetch = new IntObjectHashMap<>(numReaders);
        for (int i = 0; i < numReaders; i++) {
            int readerId = in.readVInt();
            int numDocs = in.readVInt();
            IntArrayList docs = new IntArrayList(numDocs);
            toFetch.put(readerId, docs);
            for (int j = 0; j < numDocs; j++) {
                docs.add(in.readInt());
            }
        }
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    requests = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        requests.add(TermVectorsRequest.readTermVectorsRequest(in));
    }

    preference = in.readOptionalString();
}
 

/**
 * Left divide for gallic weights (right divide is not implemented yet)
 *
 * @param a first
 * @param b second
 * @return left divide of first by second
 */
@Override
public GallicWeight divide(GallicWeight a, GallicWeight b) {
  if (isZero(a)) return zero;
  Preconditions.checkArgument(isNotZero(b), "cant divide by zero");
  IntArrayList newOutputLabels = new IntArrayList();
  for (int i = b.getLabels().size(); i < a.getLabels().size(); i++) {
    newOutputLabels.add(a.getLabels().get(i));
  }
  double newWeight = this.weightSemiring.divide(a.getWeight(), b.getWeight());
  return GallicWeight.create(newOutputLabels, newWeight);
}
 

public ShardResponse(StreamInput in) throws IOException {
    super(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            failures.add(new Failure(in));
        } else {
            failures.add(null);
        }
    }
    if (in.readBoolean()) {
        failure = in.readException();
    }
    if (in.getVersion().onOrAfter(Version.V_4_2_0)) {
        int resultColumnsSize = in.readVInt();
        if (resultColumnsSize > 0) {
            resultColumns = new Symbol[resultColumnsSize];
            for (int i = 0; i < resultColumnsSize; i++) {
                Symbol symbol = Symbols.fromStream(in);
                resultColumns[i] = symbol;
            }
            Streamer[] resultRowStreamers = Symbols.streamerArray(List.of(resultColumns));
            int resultRowsSize = in.readVInt();
            if (resultRowsSize > 0) {
                resultRows = new ArrayList<>(resultRowsSize);
                int rowLength = in.readVInt();
                for (int i = 0; i < resultRowsSize; i++) {
                    Object[] row = new Object[rowLength];
                    for (int j = 0; j < rowLength; j++) {
                        row[j] = resultRowStreamers[j].readValueFrom(in);
                    }
                    resultRows.add(row);
                }
            }
        }
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    routing = in.readOptionalString();
    jobId = new UUID(in.readLong(), in.readLong());

    int size = in.readVInt();
    locations = new IntArrayList(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
    }
}
 

private RootTask(Logger logger,
                 UUID jobId,
                 String coordinatorNodeId,
                 Collection<String> participatingNodes,
                 JobsLogs jobsLogs,
                 List<Task> orderedTasks,
                 @Nullable ProfilingContext profilingContext) throws Exception {
    this.logger = logger;
    this.coordinatorNodeId = coordinatorNodeId;
    this.participatedNodes = participatingNodes;
    this.jobId = jobId;
    this.jobsLogs = jobsLogs;

    int numTasks = orderedTasks.size();

    if (profilingContext == null) {
        taskTimersByPhaseId = null;
        profilingFuture = null;
        profiler = null;
    } else {
        profiler = profilingContext;
        taskTimersByPhaseId = new ConcurrentHashMap<>(numTasks);
        profilingFuture = new CompletableFuture<>();
    }

    orderedTaskIds = new IntArrayList(numTasks);
    tasksByPhaseId = new ConcurrentHashMap<>(numTasks);
    this.numTasks = new AtomicInteger(numTasks);

    traceEnabled = logger.isTraceEnabled();
    for (Task task : orderedTasks) {
        int phaseId = task.id();
        orderedTaskIds.add(phaseId);
        if (tasksByPhaseId.put(phaseId, task) != null) {
            throw new IllegalArgumentException("Task for " + phaseId + " already added");
        }
        task.completionFuture().whenComplete(new RemoveTaskListener(phaseId));
        jobsLogs.operationStarted(phaseId, jobId, task.name(), task::bytesUsed);
        task.prepare();
        if (profiler != null) {
            String subContextName = ProfilingContext.generateProfilingKey(task.id(), task.name());
            if (taskTimersByPhaseId.put(phaseId, profiler.createTimer(subContextName)) != null) {
                throw new IllegalArgumentException("Timer for " + phaseId + " already added");
            }
        }
        if (traceEnabled) {
            logger.trace("adding subContext {}, now there are {} tasksByPhaseId", phaseId, tasksByPhaseId.size());
        }
    }
}
 

protected void addCountsFromDocument(IntArrayList[] positions, int[] counts, int docLength) {
    if (docLength <= windowSize) {
        addCountsFromSmallDocument(positions, counts);
        return;
    }
    int posInList[] = new int[positions.length + 1];
    int nextWordId = 0,
        nextWordPos = Integer.MAX_VALUE;
    int wordCount = 0;
    // determine the first token which we should look at
    for (int i = 0; i < positions.length; ++i) {
        if (positions[i] != null) {
            Arrays.sort(positions[i].buffer, 0, positions[i].elementsCount);
            if (positions[i].buffer[0] < nextWordPos) {
                nextWordPos = positions[i].buffer[0];
                nextWordId = i;
            }
            wordCount += positions[i].elementsCount;
        }
    }

    IntArrayList wordIdsInWindow = new IntArrayList(wordCount < windowSize ? wordCount : windowSize);
    IntArrayList wordPositionsInWindow = new IntArrayList(wordCount < windowSize ? wordCount : windowSize);
    int romaveableWordsPosId = posInList.length - 1;
    int windowWords = 0;
    int lastWordPos,
        wordEndPos;
    boolean countingEnabled = false;
    while (nextWordPos < docLength) {
        // create (or udpate) a signature containing a 1 for every word type inside this window
        // check whether a word will be removed
        if (nextWordId == positions.length) {
            windowWords &= ~(1 << wordIdsInWindow.buffer[posInList[romaveableWordsPosId]]);
            ++posInList[romaveableWordsPosId];
        } else {
            // if this word is already inside the window
            if ((windowWords & (1 << nextWordId)) > 0) {
                // we have to remove its first occurrence from the list
                for (int i = posInList[romaveableWordsPosId]; i < wordIdsInWindow.elementsCount; ++i) {
                    if (wordIdsInWindow.buffer[i] == nextWordId) {
                        wordIdsInWindow.remove(i);
                        wordPositionsInWindow.remove(i);
                    }
                }
            } else {
                // add the word
                windowWords |= (1 << nextWordId);
            }
            // add its position to the list of tokens that should be removed (if this would be inside this document)
            wordEndPos = nextWordPos + windowSize;
            if (wordEndPos < docLength) {
                wordIdsInWindow.add(nextWordId);
                wordPositionsInWindow.add(wordEndPos);
            }
            // check if on the same position a word should be removed
            if ((posInList[romaveableWordsPosId] < wordPositionsInWindow.elementsCount)
                    && (wordPositionsInWindow.buffer[posInList[romaveableWordsPosId]] == nextWordPos)) {
                windowWords &= ~(1 << wordIdsInWindow.buffer[posInList[romaveableWordsPosId]]);
                ++posInList[romaveableWordsPosId];
            }
            ++posInList[nextWordId];
        }

        // Find the next position we should look at
        lastWordPos = nextWordPos;
        nextWordPos = Integer.MAX_VALUE;
        for (int i = 0; i < positions.length; ++i) {
            if ((positions[i] != null) && (posInList[i] < positions[i].elementsCount)
                    && (positions[i].buffer[posInList[i]] < nextWordPos)) {
                nextWordPos = positions[i].buffer[posInList[i]];
                nextWordId = i;
            }
        }
        if ((posInList[romaveableWordsPosId] < wordPositionsInWindow.elementsCount)
                && (wordPositionsInWindow.buffer[posInList[romaveableWordsPosId]] < nextWordPos)) {
            nextWordPos = wordPositionsInWindow.buffer[posInList[romaveableWordsPosId]];
            nextWordId = positions.length;
        }
        // Make sure that counting only starts if the first window is complete
        if ((!countingEnabled) && (nextWordPos >= windowSize)) {
            if (lastWordPos < windowSize) {
                lastWordPos = windowSize - 1;
            }
            countingEnabled = true;
        }
        if ((countingEnabled) && (windowWords != 0)) {
            // increase counts
            if (nextWordPos < docLength) {
                counts[windowWords] += nextWordPos - lastWordPos;
            } else {
                counts[windowWords] += docLength - lastWordPos;
            }
        }
    }
}
 

@Override
public void getDocumentsWithWord(String word, IntArrayList documents) {
    documents.add(wordDocuments[Integer.parseInt(word)]);
}
 

/**
 * find all phases that don't have any downstreams.
 * <p>
 * This is usually only one phase (the handlerPhase, but there might be more in case of bulk operations)
 * <pre>
 *
 *     flow:            targetToSourceMap:
 *
 *     0    1               2 -> [0, 1]
 *      \  /                3 -> [2]
 *       2
 *       |
 *       3
 *
 *     leafs = all keys in targetToSource map which have no entry in values (3 in the example above)
 * </pre>
 */
private static IntCollection findLeafs(IntObjectMap<? extends IntContainer> targetToSourceMap) {
    IntArrayList leafs = new IntArrayList();
    BitSet sources = new BitSet();
    for (IntObjectCursor<? extends IntContainer> sourceIds : targetToSourceMap) {
        for (IntCursor sourceId : sourceIds.value) {
            sources.set(sourceId.value);
        }
    }
    for (IntCursor targetPhaseCursor : targetToSourceMap.keys()) {
        int targetPhase = targetPhaseCursor.value;
        if (!sources.get(targetPhase)) {
            leafs.add(targetPhase);
        }
    }
    return leafs;
}