org.apache.lucene.util.RamUsageEstimator Java Examples

private BM25FQuery(BM25Similarity similarity, TreeMap<String, FieldAndWeight> fieldAndWeights, BytesRef[] terms) {
  this.similarity = similarity;
  this.fieldAndWeights = fieldAndWeights;
  this.terms = terms;
  int numFieldTerms = fieldAndWeights.size() * terms.length;
  if (numFieldTerms > IndexSearcher.getMaxClauseCount()) {
    throw new IndexSearcher.TooManyClauses();
  }
  this.fieldTerms = new Term[numFieldTerms];
  Arrays.sort(terms);
  int pos = 0;
  for (String field : fieldAndWeights.keySet()) {
    for (BytesRef term : terms) {
      fieldTerms[pos++] = new Term(field, term);
    }
  }

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(fieldAndWeights) +
      RamUsageEstimator.sizeOfObject(fieldTerms) +
      RamUsageEstimator.sizeOfObject(terms);
}
 

@Override
public Map<Object, Object> iterate(RamAccounting ramAccounting,
                                   MemoryManager memoryManager,
                                   Map<Object, Object> state,
                                   Input... args) throws CircuitBreakingException {
    Object value = args[0].value();
    if (value == null) {
        return state;
    }
    if (state.put(value, PRESENT) == null) {
        ramAccounting.addBytes(
            // values size + 32 bytes for entry, 4 bytes for increased capacity
            RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(value) + 36L)
        );
    }
    return state;
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newLongPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

@Override
public Map<Object, Long> removeFromAggregatedState(RamAccounting ramAccounting,
                                                   Map<Object, Long> previousAggState,
                                                   Input[] stateToRemove) {
    Object value = stateToRemove[0].value();
    if (value == null) {
        return previousAggState;
    }
    Long numTimesValueSeen = previousAggState.get(value);
    if (numTimesValueSeen == null) {
        return previousAggState;
    }
    if (numTimesValueSeen == 1) {
        previousAggState.remove(value);
        ramAccounting.addBytes(
            // we initially accounted for values size + 32 bytes for entry, 4 bytes for increased capacity
            // and 12 bytes for the array container and the int value it stored
            - RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(value) + 48L)
        );
    } else {
        previousAggState.put(value, numTimesValueSeen - 1);
    }
    return previousAggState;
}
 

/**
 * Writes all operations in the given iterable to the given output stream including the size of the array
 * use {@link #readOperations(StreamInput)} to read it back.
 */
public static void writeOperations(StreamOutput outStream, List<Operation> toWrite) throws IOException {
    final ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(BigArrays.NON_RECYCLING_INSTANCE);
    try {
        outStream.writeInt(toWrite.size());
        final BufferedChecksumStreamOutput checksumStreamOutput = new BufferedChecksumStreamOutput(out);
        for (Operation op : toWrite) {
            out.reset();
            final long start = out.position();
            out.skip(RamUsageEstimator.NUM_BYTES_INT);
            writeOperationNoSize(checksumStreamOutput, op);
            long end = out.position();
            int operationSize = (int) (out.position() - RamUsageEstimator.NUM_BYTES_INT - start);
            out.seek(start);
            out.writeInt(operationSize);
            out.seek(end);
            ReleasablePagedBytesReference bytes = out.bytes();
            bytes.writeTo(outStream);
        }
    } finally {
        Releasables.close(out.bytes());
    }

}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newBytePage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

private void incr() {
  upto++;
  grow();
  if (arcs.length <= upto) {
    @SuppressWarnings({"rawtypes","unchecked"}) final FST.Arc<T>[] newArcs =
      new FST.Arc[ArrayUtil.oversize(1+upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(arcs, 0, newArcs, 0, arcs.length);
    arcs = newArcs;
  }
  if (output.length <= upto) {
    @SuppressWarnings({"rawtypes","unchecked"}) final T[] newOutput =
      (T[]) new Object[ArrayUtil.oversize(1+upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(output, 0, newOutput, 0, output.length);
    output = newOutput;
  }
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newObjectPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

/**
 * @param toField               The field that should contain terms that are specified in the next parameter.
 * @param terms                 The terms that matching documents should have. The terms must be sorted by natural order.
 * @param indexReaderContextId  Refers to the top level index reader used to create the set of terms in the previous parameter.
 */
TermsQuery(String toField, BytesRefHash terms, String fromField, Query fromQuery, Object indexReaderContextId) {
  super(toField);
  this.terms = terms;
  ords = terms.sort();
  this.fromField = fromField;
  this.fromQuery = fromQuery;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(field) +
      RamUsageEstimator.sizeOfObject(fromField) +
      RamUsageEstimator.sizeOfObject(fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(ords) +
      RamUsageEstimator.sizeOfObject(terms);
}
 

GlobalOrdinalsWithScoreQuery(GlobalOrdinalsWithScoreCollector collector, ScoreMode scoreMode, String joinField,
                             OrdinalMap globalOrds, Query toQuery, Query fromQuery, int min, int max,
                             Object indexReaderContextId) {
  this.collector = collector;
  this.joinField = joinField;
  this.globalOrds = globalOrds;
  this.toQuery = toQuery;
  this.scoreMode = scoreMode;
  this.fromQuery = fromQuery;
  this.min = min;
  this.max = max;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(this.fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(this.globalOrds) +
      RamUsageEstimator.sizeOfObject(this.joinField) +
      RamUsageEstimator.sizeOfObject(this.toQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 

/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 

@Override
public DeltaPackedLongValues build() {
  finish();
  pending = null;
  final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff);
  final long[] mins = ArrayUtil.copyOfSubArray(this.mins, 0, valuesOff);
  final long ramBytesUsed = DeltaPackedLongValues.BASE_RAM_BYTES_USED
      + RamUsageEstimator.sizeOf(values) + RamUsageEstimator.sizeOf(mins);
  return new DeltaPackedLongValues(pageShift, pageMask, values, mins, size, ramBytesUsed);
}
 

@Override
public long ramBytesUsed() {
  long size = BASE_RAM_BYTES_USED;
  for (byte[] block : blocks) {
    size += RamUsageEstimator.sizeOf(block);
  }
  return size;
}
 

@Override
public long ramBytesUsed() {
  return RamUsageEstimator.alignObjectSize(
      RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
      + RamUsageEstimator.NUM_BYTES_OBJECT_REF
      + Long.BYTES
      + Float.BYTES)
      + current.ramBytesUsed();
}
 

@Override
public MonotonicLongValues build() {
  finish();
  pending = null;
  final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff);
  final long[] mins = ArrayUtil.copyOfSubArray(this.mins, 0, valuesOff);
  final float[] averages = ArrayUtil.copyOfSubArray(this.averages, 0, valuesOff);
  final long ramBytesUsed = MonotonicLongValues.BASE_RAM_BYTES_USED
      + RamUsageEstimator.sizeOf(values) + RamUsageEstimator.sizeOf(mins)
      + RamUsageEstimator.sizeOf(averages);
  return new MonotonicLongValues(pageShift, pageMask, values, mins, averages, size, ramBytesUsed);
}
 

@Nullable
@Override
public Map<Object, Long> newState(RamAccounting ramAccounting,
                                  Version indexVersionCreated,
                                  Version minNodeInCluster,
                                  MemoryManager memoryManager) {
    ramAccounting.addBytes(RamUsageEstimator.alignObjectSize(64L)); // overhead for HashMap: 32 * 0 + 16 * 4 bytes
    return new HashMap<>();
}
 

@Override
public long ramBytesUsed() {
  long mem = RamUsageEstimator.shallowSizeOf(this);
  try {
    if (searcherMgr != null) {
      SearcherManager mgr;
      IndexSearcher searcher;
      synchronized (searcherMgrLock) {
        mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
        searcher = mgr.acquire();
      }
      try {
        for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
          LeafReader reader = FilterLeafReader.unwrap(context.reader());
          if (reader instanceof SegmentReader) {
            mem += ((SegmentReader) context.reader()).ramBytesUsed();
          }
        }
      } finally {
        mgr.release(searcher);
      }
    }
    return mem;
  } catch (IOException ioe) {
    throw new RuntimeException(ioe);
  }
}
 

public QueryResultKey(Query query, List<Query> filters, Sort sort, int nc_flags, int minExactCount) {
  this.query = query;
  this.sort = sort;
  this.filters = filters;
  this.nc_flags = nc_flags;
  this.minExactCount = minExactCount;

  int h = query.hashCode();

  if (filters != null) {
    for (Query filt : filters)
      // NOTE: simple summation used here so keys with the same filters but in
      // different orders get the same hashCode
      h += filt.hashCode();
  }

  sfields = (this.sort !=null) ? this.sort.getSort() : defaultSort;
  long ramSfields = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
  for (SortField sf : sfields) {
    h = h*29 + sf.hashCode();
    ramSfields += BASE_SF_RAM_BYTES_USED + RamUsageEstimator.sizeOfObject(sf.getField());
  }
  h = h*31 + minExactCount;

  hc = h;

  ramBytesUsed =
      BASE_RAM_BYTES_USED +
      ramSfields +
      RamUsageEstimator.sizeOfObject(query, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(filters, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 

@Override
public long ramBytesUsed() {
  return RamUsageEstimator.alignObjectSize(
        RamUsageEstimator.NUM_BYTES_OBJECT_REF
      + Integer.BYTES)
      + intSet.ramBytesUsed();
}
 

@Override
public long ramBytesUsed() {
  return BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(field) +
      RamUsageEstimator.sizeOfObject(shape) +
      RamUsageEstimator.sizeOfObject(shapeBounds);
}
 

@Override
public String toString() {
  return "SortedIntDocSet{" +
      "size=" + size() + "," +
      "ramUsed=" + RamUsageEstimator.humanReadableUnits(ramBytesUsed())+
      '}';
}
 

/**
 * Primary constructor for a DocSlice instance.
 *
 * @param offset  starting offset for this range of docs
 * @param len     length of results
 * @param docs    array of docids starting at position 0
 * @param scores  array of scores that corresponds to docs, may be null
 * @param matches total number of matches for the query
 * @param matchesRelation Indicates if {@code matches} is exact or an approximation
 */
public DocSlice(int offset, int len, int[] docs, float[] scores, long matches, float maxScore, TotalHits.Relation matchesRelation) {
  this.offset=offset;
  this.len=len;
  this.docs=docs;
  this.scores=scores;
  this.matches=matches;
  this.maxScore=maxScore;
  this.ramBytesUsed = BASE_RAM_BYTES_USED + (docs == null ? 0 : ((long)docs.length << 2)) + (scores == null ? 0 : ((long)scores.length<<2)+RamUsageEstimator.NUM_BYTES_ARRAY_HEADER);
  this.matchesRelation = matchesRelation;
}
 

@Override
public long ramBytesUsed() {
  // Return a rough estimation for allocated blocks. Note that we do not make
  // any special distinction for direct memory buffers.
  return RamUsageEstimator.NUM_BYTES_OBJECT_REF * blocks.size() + 
         blocks.stream().mapToLong(buf -> buf.capacity()).sum();
}
 

@Override
public void run() {
    long startTimeNs = System.nanoTime();
    long startMeasurements = startTimeNs + warmUpNs;

    // 堆大小
    long memoryKb = RamUsageEstimator.shallowSizeOf(testInstance);
    statisticsCalculator.setMemory(memoryKb);

    while (isContinue) {
        evaluateStatement(startMeasurements);
    }
}
 

private static <T> T[] grow(T[] array, int minSize) {
    if (array.length < minSize) {
        final int newLen = ArrayUtil.oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
        array = Arrays.copyOf(array, newLen);
    }
    return array;
}
 

/** Build a {@link PackedLongValues} instance that contains values that
 *  have been added to this builder. This operation is destructive. */
public PackedLongValues build() {
  finish();
  pending = null;
  final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff);
  final long ramBytesUsed = PackedLongValues.BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values);
  return new PackedLongValues(pageShift, pageMask, values, size, ramBytesUsed);
}
 

@Nullable
@Override
public Map<Object, Object> newState(RamAccounting ramAccounting,
                                    Version indexVersionCreated,
                                    Version minNodeInCluster,
                                    MemoryManager memoryManager) {
    ramAccounting.addBytes(RamUsageEstimator.alignObjectSize(64L)); // overhead for HashMap: 32 * 0 + 16 * 4 bytes
    return new HashMap<>();
}
 

private FST.Arc<BytesRef> getArc(int ord) {
  if (ord >= arcs.length) {
    @SuppressWarnings({"rawtypes","unchecked"}) final FST.Arc<BytesRef>[] next =
      new FST.Arc[ArrayUtil.oversize(1+ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(arcs, 0, next, 0, arcs.length);
    for(int arcOrd=arcs.length;arcOrd<next.length;arcOrd++) {
      next[arcOrd] = new FST.Arc<BytesRef>();
    }
    arcs = next;
  }
  return arcs[ord];
}