org.apache.lucene.util.ArrayUtil Java Examples

void startBlock(FieldReader.SegmentTermsEnum.Frame frame, boolean isFloor) {
  totalBlockCount++;
  if (isFloor) {
    if (frame.fp == frame.fpOrig) {
      floorBlockCount++;
    }
    floorSubBlockCount++;
  } else {
    nonFloorBlockCount++;
  }

  if (blockCountByPrefixLen.length <= frame.prefix) {
    blockCountByPrefixLen = ArrayUtil.grow(blockCountByPrefixLen, 1+frame.prefix);
  }
  blockCountByPrefixLen[frame.prefix]++;
  startBlockCount++;
  totalBlockSuffixBytes += frame.suffixesReader.length();
  totalBlockStatsBytes += frame.statsReader.length();
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newBytePage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

public static void verifyParams(int numDims, int numIndexDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
  // We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use
  // remaining 4 bits for another purpose later
  if (numDims < 1 || numDims > MAX_DIMS) {
    throw new IllegalArgumentException("numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");
  }
  if (numIndexDims < 1 || numIndexDims > MAX_INDEX_DIMS) {
    throw new IllegalArgumentException("numIndexDims must be 1 .. " + MAX_INDEX_DIMS + " (got: " + numIndexDims + ")");
  }
  if (numIndexDims > numDims) {
    throw new IllegalArgumentException("numIndexDims cannot exceed numDims (" + numDims + ") (got: " + numIndexDims + ")");
  }
  if (maxPointsInLeafNode <= 0) {
    throw new IllegalArgumentException("maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
  }
  if (maxPointsInLeafNode > ArrayUtil.MAX_ARRAY_LENGTH) {
    throw new IllegalArgumentException("maxPointsInLeafNode must be <= ArrayUtil.MAX_ARRAY_LENGTH (= " + ArrayUtil.MAX_ARRAY_LENGTH + "); got " + maxPointsInLeafNode);
  }
  if (maxMBSortInHeap < 0.0) {
    throw new IllegalArgumentException("maxMBSortInHeap must be >= 0.0 (got: " + maxMBSortInHeap + ")");
  }
  if (totalPointCount < 0) {
    throw new IllegalArgumentException("totalPointCount must be >=0 (got: " + totalPointCount + ")");
  }
}
 

/** Creates a new iterator, buffering entries from the specified iterator */
public BufferedInputIterator(InputIterator source) throws IOException {
  BytesRef spare;
  int freqIndex = 0;
  hasPayloads = source.hasPayloads();
  hasContexts = source.hasContexts();
  while((spare = source.next()) != null) {
    entries.append(spare);
    if (hasPayloads) {
      payloads.append(source.payload());
    }
    if (hasContexts) {
      contextSets.add(source.contexts());
    }
    if (freqIndex >= freqs.length) {
      freqs = ArrayUtil.grow(freqs, freqs.length+1);
    }
    freqs[freqIndex++] = source.weight();
  }
 
}
 

@Test
public void testLargeArrayAdd() {
  ByteBuffersDataOutput o = new ByteBuffersDataOutput();
  int MB = 1024 * 1024;
  final byte [] bytes;
  if (LuceneTestCase.TEST_NIGHTLY) {
    bytes = randomBytesOfLength(5 * MB, 15 * MB);
  } else {
    bytes = randomBytesOfLength(MB/2, MB);
  }
  int offset = randomIntBetween(0, 100);
  int len = bytes.length - offset;
  o.writeBytes(bytes, offset, len);
  assertEquals(len, o.size());
  Assert.assertArrayEquals(ArrayUtil.copyOfSubArray(bytes, offset, offset + len), o.toArrayCopy());
}
 

/**
 * Build a minimal, deterministic automaton from a sorted list of {@link BytesRef} representing
 * strings in UTF-8. These strings must be binary-sorted.
 */
public static Automaton build(Collection<BytesRef> input) {
  final DaciukMihovAutomatonBuilder builder = new DaciukMihovAutomatonBuilder();
  
  char[] chars = new char[0];
  CharsRef ref = new CharsRef();
  for (BytesRef b : input) {
    chars = ArrayUtil.grow(chars, b.length);
    final int len = UnicodeUtil.UTF8toUTF16(b, chars);
    ref.chars = chars;
    ref.length = len;
    builder.add(ref);
  }
  
  Automaton.Builder a = new Automaton.Builder();
  convert(a,
      builder.complete(), 
      new IdentityHashMap<State,Integer>());

  return a.finish();
}
 

/**
 * Adjusts <code>expDocNrs</code> based on the filler docs injected in the index, 
 * and if neccessary wraps the <code>q</code> in a BooleanQuery that will filter out all 
 * filler docs using the {@link #EXTRA} field.
 * 
 * @see #replaceIndex
 */
@Override
public void qtest(Query q, int[] expDocNrs) throws Exception {

  expDocNrs = ArrayUtil.copyOfSubArray(expDocNrs, 0, expDocNrs.length);
  for (int i=0; i < expDocNrs.length; i++) {
    expDocNrs[i] = PRE_FILLER_DOCS + ((NUM_FILLER_DOCS + 1) * expDocNrs[i]);
  }

  if (null != EXTRA) {
    BooleanQuery.Builder builder = new BooleanQuery.Builder();
    builder.add(new BooleanClause(q, BooleanClause.Occur.MUST));
    builder.add(new BooleanClause(new TermQuery(new Term(EXTRA, EXTRA)), BooleanClause.Occur.MUST_NOT));
    q = builder.build();
  }
  super.qtest(q, expDocNrs);
}
 

@Override
public void setDocument(int docId) {
    bytes = values.get(docId);
    in.reset(bytes.bytes, bytes.offset, bytes.length);
    if (!in.eof()) {
        // first value uses vLong on top of zig-zag encoding, then deltas are encoded using vLong
        long previousValue = longs[0] = ByteUtils.zigZagDecode(ByteUtils.readVLong(in));
        count = 1;
        while (!in.eof()) {
            longs = ArrayUtil.grow(longs, count + 1);
            previousValue = longs[count++] = previousValue + ByteUtils.readVLong(in);
        }
    } else {
        count = 0;
    }
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newBytePage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newObjectPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newLongPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

DirectMonotonicWriter(IndexOutput metaOut, IndexOutput dataOut, long numValues, int blockShift) {
  if (blockShift < MIN_BLOCK_SHIFT || blockShift > MAX_BLOCK_SHIFT) {
    throw new IllegalArgumentException("blockShift must be in [" + MIN_BLOCK_SHIFT + "-" + MAX_BLOCK_SHIFT + "], got " + blockShift);
  }
  if (numValues < 0) {
    throw new IllegalArgumentException("numValues can't be negative, got " + numValues);
  }
  final long numBlocks = numValues == 0 ? 0 : ((numValues - 1) >>> blockShift) + 1;
  if (numBlocks > ArrayUtil.MAX_ARRAY_LENGTH) {
    throw new IllegalArgumentException("blockShift is too low for the provided number of values: blockShift=" + blockShift +
        ", numValues=" + numValues + ", MAX_ARRAY_LENGTH=" + ArrayUtil.MAX_ARRAY_LENGTH);
  }
  this.meta = metaOut;
  this.data = dataOut;
  this.numValues = numValues;
  final int blockSize = 1 << blockShift;
  this.buffer = new long[(int) Math.min(numValues, blockSize)];
  this.bufferSize = 0;
  this.baseDataPointer = dataOut.getFilePointer();
}
 

private void checkReadBytes(IndexInput input, int size, int pos) throws IOException{
  // Just to see that "offset" is treated properly in readBytes(), we
  // add an arbitrary offset at the beginning of the array
  int offset = size % 10; // arbitrary
  buffer = ArrayUtil.grow(buffer, offset+size);
  assertEquals(pos, input.getFilePointer());
  long left = TEST_FILE_LENGTH - input.getFilePointer();
  if (left <= 0) {
    return;
  } else if (left < size) {
    size = (int) left;
  }
  input.readBytes(buffer, offset, size);
  assertEquals(pos+size, input.getFilePointer());
  for(int i=0; i<size; i++) {
    assertEquals("pos=" + i + " filepos=" + (pos+i), byten(pos+i), buffer[offset+i]);
  }
}
 

@Override
public Optional<SearchResults> prevPage() {
  if (currentPage < 0 || query == null) {
    throw new LukeException(new IllegalStateException("Search session not started."));
  }

  // return to previous page
  currentPage -= 1;

  if (currentPage < 0) {
    log.warn("No more previous search results are available.");
    return Optional.empty();
  }

  try {
    // there should be cached results for this page
    int from = currentPage * pageSize;
    int to = Math.min(from + pageSize, docs.length);
    ScoreDoc[] part = ArrayUtil.copyOfSubArray(docs, from, to);
    return Optional.of(SearchResults.of(totalHits, part, from, searcher, fieldsToLoad));
  } catch (IOException e) {
    throw new LukeException("Search Failed.", e);
  }
}
 

public DocValuesTermsQuery(String field, Collection<BytesRef> terms) {
  this.field = Objects.requireNonNull(field);
  Objects.requireNonNull(terms, "Collection of terms must not be null");
  BytesRef[] sortedTerms = terms.toArray(new BytesRef[terms.size()]);
  ArrayUtil.timSort(sortedTerms);
  PrefixCodedTerms.Builder builder = new PrefixCodedTerms.Builder();
  BytesRef previous = null;
  for (BytesRef term : sortedTerms) {
    if (term.equals(previous) == false) {
      builder.add(field, term);
    }
    previous = term;
  }
  termData = builder.finish();
  termDataHashCode = termData.hashCode();
}
 

@Override
public void collect(int doc) throws IOException {
  if (docValues.advanceExact(doc)) {
    long ord;
    while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      int termID = collectedTerms.add(docValues.lookupOrd(ord));
      if (termID < 0) {
        termID = -termID - 1;
      } else {
        if (termID >= scoreSums.length) {
          scoreSums = ArrayUtil.grow(scoreSums);
          scoreCounts = ArrayUtil.grow(scoreCounts);
        }
      }
    
      scoreSums[termID] += scorer.score();
      scoreCounts[termID]++;
    }
  }
}
 

@Override
public int nextPosition() throws IOException {
  final int token = postingInput.readVInt();
  pos += token >>> 1;
  if (storeOffsets) {
    startOffset = endOffset + postingInput.readVInt();
    endOffset = startOffset + postingInput.readVInt();
  }
  if ((token & 1) != 0) {
    payload.offset = 0;
    payload.length = postingInput.readVInt();
    if (payload.length > payload.bytes.length) {
      payload.bytes = new byte[ArrayUtil.oversize(payload.length, 1)];
    }
    postingInput.readBytes(payload.bytes, 0, payload.length);
  } else {
    payload.length = 0;
  }
  return pos;
}
 

private void reserve(int n) {
  if (capacity < n) {
    int oversize = ArrayUtil.oversize(n, Integer.BYTES);
    nodeDicType = new Type[oversize];
    nodeWordID = new int[oversize];
    nodeMark = new int[oversize];
    nodeLeftID = new int[oversize];
    nodeRightID = new int[oversize];
    nodeWordCost = new int[oversize];
    nodeLeftCost = new int[oversize];
    nodeRightCost = new int[oversize];
    nodeLeftNode = new int[oversize];
    nodeRightNode = new int[oversize];
    nodeLeft = new int[oversize];
    nodeRight = new int[oversize];
    nodeLeftChain = new int[oversize];
    nodeRightChain = new int[oversize];
    capacity = oversize;
  }
}
 

void addMapping(int sourceId, int wordId) {
  if (wordId <= lastWordId) {
    throw new IllegalStateException("words out of order: " + wordId + " vs lastID: " + lastWordId);
  }
  
  if (sourceId > lastSourceId) {
    targetMapOffsets = ArrayUtil.grow(targetMapOffsets, sourceId + 1);
    for (int i = lastSourceId + 1; i <= sourceId; i++) {
      targetMapOffsets[i] = targetMapEndOffset;
    }
  } else if (sourceId != lastSourceId) {
    throw new IllegalStateException("source ids not in increasing order: lastSourceId=" + lastSourceId + " vs sourceId=" + sourceId);
  }

  targetMap = ArrayUtil.grow(targetMap, targetMapEndOffset + 1);
  targetMap[targetMapEndOffset] = wordId;
  targetMapEndOffset++;

  lastSourceId = sourceId;
  lastWordId = wordId;
}
 

/**
 * Saves the existing attribute states
 */
private void saveState() {
    // otherwise, we have delimiters, save state
    savedStartOffset = offsetAttribute.startOffset();
    savedEndOffset = offsetAttribute.endOffset();
    // if length by start + end offsets doesn't match the term text then assume this is a synonym and don't adjust the offsets.
    hasIllegalOffsets = savedEndOffset - savedStartOffset != termAttribute.length();
    savedType = typeAttribute.type();

    if (savedBuffer.length < termAttribute.length()) {
        savedBuffer = new char[ArrayUtil.oversize(termAttribute.length(), Character.BYTES)];
    }

    System.arraycopy(termAttribute.buffer(), 0, savedBuffer, 0, termAttribute.length());
    iterator.text = savedBuffer;

    hasSavedState = true;
}
 

private void addOneValue(BytesRef value) {
  int termID = hash.add(value);
  if (termID < 0) {
    termID = -termID-1;
  } else {
    // reserve additional space for each unique value:
    // 1. when indexing, when hash is 50% full, rehash() suddenly needs 2*size ints.
    //    TODO: can this same OOM happen in THPF?
    // 2. when flushing, we need 1 int per value (slot in the ordMap).
    iwBytesUsed.addAndGet(2 * Integer.BYTES);
  }
  
  if (currentUpto == currentValues.length) {
    currentValues = ArrayUtil.grow(currentValues, currentValues.length+1);
    iwBytesUsed.addAndGet((currentValues.length - currentUpto) * Integer.BYTES);
  }
  
  currentValues[currentUpto] = termID;
  currentUpto++;
}
 

public static void verifyParams(int numDims, int numIndexDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
  // We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use
  // remaining 4 bits for another purpose later
  if (numDims < 1 || numDims > MAX_DIMS) {
    throw new IllegalArgumentException("numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");
  }
  if (numIndexDims < 1 || numIndexDims > MAX_INDEX_DIMS) {
    throw new IllegalArgumentException("numIndexDims must be 1 .. " + MAX_INDEX_DIMS + " (got: " + numIndexDims + ")");
  }
  if (numIndexDims > numDims) {
    throw new IllegalArgumentException("numIndexDims cannot exceed numDims (" + numDims + ") (got: " + numIndexDims + ")");
  }
  if (maxPointsInLeafNode <= 0) {
    throw new IllegalArgumentException("maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
  }
  if (maxPointsInLeafNode > ArrayUtil.MAX_ARRAY_LENGTH) {
    throw new IllegalArgumentException("maxPointsInLeafNode must be <= ArrayUtil.MAX_ARRAY_LENGTH (= " + ArrayUtil.MAX_ARRAY_LENGTH + "); got " + maxPointsInLeafNode);
  }
  if (maxMBSortInHeap < 0.0) {
    throw new IllegalArgumentException("maxMBSortInHeap must be >= 0.0 (got: " + maxMBSortInHeap + ")");
  }
  if (totalPointCount < 0) {
    throw new IllegalArgumentException("totalPointCount must be >=0 (got: " + totalPointCount + ")");
  }
}
 

@Override
protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, BytesRef payload, Set<BytesRef> contexts, long weight) throws IOException {
  if (spare.length + 4 >= buffer.length) {
    buffer = ArrayUtil.grow(buffer, spare.length + 4);
  }
  output.reset(buffer);
  output.writeBytes(spare.bytes, spare.offset, spare.length);
  output.writeInt(encodeWeight(weight));
  writer.write(buffer, 0, output.getPosition());
}
 

public void grow() {
  costs = ArrayUtil.grow(costs, 1+count);
  lastRightID = ArrayUtil.grow(lastRightID, 1+count);
  backPos = ArrayUtil.grow(backPos, 1+count);
  backIndex = ArrayUtil.grow(backIndex, 1+count);
  backID = ArrayUtil.grow(backID, 1+count);

  // NOTE: sneaky: grow separately because
  // ArrayUtil.grow will otherwise pick a different
  // length than the int[]s we just grew:
  final Type[] newBackType = new Type[backID.length];
  System.arraycopy(backType, 0, newBackType, 0, backType.length);
  backType = newBackType;
}
 

@Override
void grow(int newBlockCount) {
  super.grow(newBlockCount);
  ramBytesUsed -= RamUsageEstimator.sizeOf(mins);
  mins = ArrayUtil.growExact(mins, newBlockCount);
  ramBytesUsed += RamUsageEstimator.sizeOf(mins);
}
 

public void setFloorData(ByteArrayDataInput in, BytesRef source) {
  final int numBytes = source.length - (in.getPosition() - source.offset);
  if (numBytes > floorData.length) {
    floorData = new byte[ArrayUtil.oversize(numBytes, 1)];
  }
  System.arraycopy(source.bytes, source.offset+in.getPosition(), floorData, 0, numBytes);
  floorDataReader.reset(floorData, 0, numBytes);
  numFollowFloorBlocks = floorDataReader.readVInt();
  nextFloorLabel = floorDataReader.readByte() & 0xff;
  //if (DEBUG) {
  //System.out.println("    setFloorData fpOrig=" + fpOrig + " bytes=" + new BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks=" + numFollowFloorBlocks + " nextFloorLabel=" + toHex(nextFloorLabel));
  //}
}
 

@Override
public final void collect(int doc) throws IOException {
  docsBuilder.grow(1).add(doc);
  if (keepScores) {
    if (totalHits >= scores.length) {
      float[] newScores = new float[ArrayUtil.oversize(totalHits + 1, 4)];
      System.arraycopy(scores, 0, newScores, 0, totalHits);
      scores = newScores;
    }
    scores[totalHits] = scorer.score();
  }
  totalHits++;
}
 

/**
 * Make sure the {@link #values} array can store at least {@link #count} entries.
 */
protected final void grow() {
    if (values.length < count) {
        final int oldLen = values.length;
        final int newLen = ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
        values = Arrays.copyOf(values, newLen);
        for (int i = oldLen; i < newLen; ++i) {
            values[i] = new BytesRefBuilder();
        }
    }
}
 

@Test
public void testWriteByteBuffer() {
  ByteBuffersDataOutput o = new ByteBuffersDataOutput();
  byte[] bytes = randomBytesOfLength(1024 * 8 + 10);
  ByteBuffer src = ByteBuffer.wrap(bytes);
  int offset = randomIntBetween(0, 100);
  int len = bytes.length - offset;
  src.position(offset);
  src.limit(offset + len);
  o.writeBytes(src);
  assertEquals(len, o.size());
  Assert.assertArrayEquals(ArrayUtil.copyOfSubArray(bytes, offset, offset + len), o.toArrayCopy());
}
 

@Override
public void collect(int doc) throws IOException {
  if (docValues.advanceExact(doc)) {
    long ord;
    while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      int termID = collectedTerms.add(docValues.lookupOrd(ord));
      if (termID < 0) {
        termID = -termID - 1;
      } else {
        if (termID >= scoreSums.length) {
          int begin = scoreSums.length;
          scoreSums = ArrayUtil.grow(scoreSums);
          if (scoreMode == ScoreMode.Min) {
            Arrays.fill(scoreSums, begin, scoreSums.length, Float.POSITIVE_INFINITY);
          } else if (scoreMode == ScoreMode.Max) {
            Arrays.fill(scoreSums, begin, scoreSums.length, Float.NEGATIVE_INFINITY);
          }
        }
      }
    
      switch (scoreMode) {
      case Total:
        scoreSums[termID] += scorer.score();
        break;
      case Min:
        scoreSums[termID] = Math.min(scoreSums[termID], scorer.score());
        break;
      case Max:
        scoreSums[termID] = Math.max(scoreSums[termID], scorer.score());
        break;
      default:
        throw new AssertionError("unexpected: " + scoreMode);
      }
    }
  }
}