org.apache.lucene.index.StoredFieldVisitor Java Examples

@Override
public void document(final int docID, final StoredFieldVisitor visitor) throws IOException {

    if(complianceConfig.readHistoryEnabledForIndex(indexService.index().getName())) {
        final ComplianceAwareStoredFieldVisitor cv = new ComplianceAwareStoredFieldVisitor(visitor);

        if(flsEnabled) {
            in.document(docID, new FlsStoredFieldVisitor(maskFields?new HashingStoredFieldVisitor(cv):cv));
        } else {
            in.document(docID, maskFields?new HashingStoredFieldVisitor(cv):cv);
        }

        cv.finished();
    } else {
        if(flsEnabled) {
            in.document(docID, new FlsStoredFieldVisitor(maskFields?new HashingStoredFieldVisitor(visitor):visitor));
        } else {
            in.document(docID, maskFields?new HashingStoredFieldVisitor(visitor):visitor);
        }
    }

}
 

@Override
public StoredFieldsReader getFieldsReader() {
    StoredFieldsReader fieldsReader = super.getFieldsReader();
    return new FilterStoredFieldsReader(fieldsReader) {
        @Override
        public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
            if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
                super.visitDocument(docID, visitor);
            } else {
                super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
                    @Override
                    public Status needsField(FieldInfo fieldInfo) throws IOException {
                        if (recoverySourceField.equals(fieldInfo.name)) {
                            return Status.NO;
                        }
                        return super.needsField(fieldInfo);
                    }
                });
            }
        }
    };
}
 

@Override
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
    if (docID != 0) {
        throw new IllegalArgumentException("no such doc ID " + docID);
    }
    if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) {
        assert operation.source().toBytesRef().offset == 0;
        assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length;
        visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes);
    }
    if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) {
        visitor.stringField(FAKE_ROUTING_FIELD, operation.routing().getBytes(StandardCharsets.UTF_8));
    }
    if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) {
        BytesRef bytesRef = Uid.encodeId(operation.id());
        byte[] id = new byte[bytesRef.length];
        System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length);
        visitor.stringField(FAKE_ID_FIELD, id);
    }
}
 

private void readField(BytesRef type, FieldInfo fieldInfo, StoredFieldVisitor visitor) throws IOException {
  readLine();
  assert StringHelper.startsWith(scratch.get(), VALUE);
  if (type == TYPE_STRING) {
    byte[] bytes = new byte[scratch.length() - VALUE.length];
    System.arraycopy(scratch.bytes(), VALUE.length, bytes, 0, bytes.length);
    visitor.stringField(fieldInfo, new String(bytes, 0, bytes.length, StandardCharsets.UTF_8));
  } else if (type == TYPE_BINARY) {
    byte[] copy = new byte[scratch.length()-VALUE.length];
    System.arraycopy(scratch.bytes(), VALUE.length, copy, 0, copy.length);
    visitor.binaryField(fieldInfo, copy);
  } else if (type == TYPE_INT) {
    scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length);
    visitor.intField(fieldInfo, Integer.parseInt(scratchUTF16.toString()));
  } else if (type == TYPE_LONG) {
    scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length);
    visitor.longField(fieldInfo, Long.parseLong(scratchUTF16.toString()));
  } else if (type == TYPE_FLOAT) {
    scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length);
    visitor.floatField(fieldInfo, Float.parseFloat(scratchUTF16.toString()));
  } else if (type == TYPE_DOUBLE) {
    scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length);
    visitor.doubleField(fieldInfo, Double.parseDouble(scratchUTF16.toString()));
  }
}
 

private static void readField(DataInput in, StoredFieldVisitor visitor, FieldInfo info, int bits) throws IOException {
  switch (bits & TYPE_MASK) {
    case BYTE_ARR:
      int length = in.readVInt();
      byte[] data = new byte[length];
      in.readBytes(data, 0, length);
      visitor.binaryField(info, data);
      break;
    case STRING:
      visitor.stringField(info, in.readString());
      break;
    case NUMERIC_INT:
      visitor.intField(info, in.readZInt());
      break;
    case NUMERIC_FLOAT:
      visitor.floatField(info, readZFloat(in));
      break;
    case NUMERIC_LONG:
      visitor.longField(info, readTLong(in));
      break;
    case NUMERIC_DOUBLE:
      visitor.doubleField(info, readZDouble(in));
      break;
    default:
      throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
  }
}
 

@Override
public void visitDocument(int docID, StoredFieldVisitor visitor)
    throws IOException {

  final SerializedDocument doc = document(docID);

  for (int fieldIDX = 0; fieldIDX < doc.numStoredFields; fieldIDX++) {
    final long infoAndBits = doc.in.readVLong();
    final int fieldNumber = (int) (infoAndBits >>> TYPE_BITS);
    final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);

    final int bits = (int) (infoAndBits & TYPE_MASK);
    assert bits <= NUMERIC_DOUBLE: "bits=" + Integer.toHexString(bits);

    switch(visitor.needsField(fieldInfo)) {
      case YES:
        readField(doc.in, visitor, fieldInfo, bits);
        break;
      case NO:
        if (fieldIDX == doc.numStoredFields - 1) {// don't skipField on last field value; treat like STOP
          return;
        }
        skipField(doc.in, bits);
        break;
      case STOP:
        return;
    }
  }
}
 

private static AtomicReader setDocSize(AtomicReader reader, final int count) {
  return new FilterAtomicReader(reader) {
    @Override
    public Bits getLiveDocs() {
      return new Bits() {
        @Override
        public boolean get(int index) {
          return true;
        }

        @Override
        public int length() {
          return count;
        }
      };
    }

    @Override
    public int numDocs() {
      return count;
    }

    @Override
    public int maxDoc() {
      return count;
    }

    @Override
    public void document(int docID, StoredFieldVisitor visitor) throws IOException {
      // Do nothing
    }
  };
}
 

public static ResetableDocumentStoredFieldVisitor getFieldSelector(final Selector selector) {
  return new ResetableDocumentStoredFieldVisitor() {
    @Override
    public Status needsField(FieldInfo fieldInfo) throws IOException {
      if (ROW_ID.equals(fieldInfo.name)) {
        return StoredFieldVisitor.Status.YES;
      }
      if (RECORD_ID.equals(fieldInfo.name)) {
        return StoredFieldVisitor.Status.YES;
      }
      if (PRIME_DOC.equals(fieldInfo.name)) {
        return StoredFieldVisitor.Status.NO;
      }
      if (FAMILY.equals(fieldInfo.name)) {
        return StoredFieldVisitor.Status.YES;
      }
      if (selector.columnFamiliesToFetch == null && selector.columnsToFetch == null) {
        return StoredFieldVisitor.Status.YES;
      }
      String columnFamily = getColumnFamily(fieldInfo.name);
      if (selector.columnFamiliesToFetch != null) {
        if (selector.columnFamiliesToFetch.contains(columnFamily)) {
          return StoredFieldVisitor.Status.YES;
        }
      }
      String columnName = getColumnName(fieldInfo.name);
      if (selector.columnsToFetch != null) {
        Set<String> columns = selector.columnsToFetch.get(columnFamily);
        if (columns != null && columns.contains(columnName)) {
          return StoredFieldVisitor.Status.YES;
        }
      }
      return StoredFieldVisitor.Status.NO;
    }

  };
}
 

/** @see SolrIndexSearcher#doc(int, StoredFieldVisitor) */
public void doc(int docId, StoredFieldVisitor visitor) throws IOException {
  if (documentCache != null) {
    // get cached document or retrieve it including all fields (and cache it)
    Document cached = doc(docId);
    visitFromCached(cached, visitor);
  } else {
    searcher.getIndexReader().document(docId, visitor);
  }
}
 

/** Executes a stored field visitor against a hit from the document cache */
private void visitFromCached(Document document, StoredFieldVisitor visitor) throws IOException {
  for (IndexableField f : document) {
    final FieldInfo info = searcher.getFieldInfos().fieldInfo(f.name());
    final StoredFieldVisitor.Status needsField = visitor.needsField(info);
    if (needsField == StoredFieldVisitor.Status.STOP) return;
    if (needsField == StoredFieldVisitor.Status.NO) continue;
    BytesRef binaryValue = f.binaryValue();
    if (binaryValue != null) {
      visitor.binaryField(info, toByteArrayUnwrapIfPossible(binaryValue));
      continue;
    }
    Number numericValue = f.numericValue();
    if (numericValue != null) {
      if (numericValue instanceof Double) {
        visitor.doubleField(info, numericValue.doubleValue());
      } else if (numericValue instanceof Integer) {
        visitor.intField(info, numericValue.intValue());
      } else if (numericValue instanceof Float) {
        visitor.floatField(info, numericValue.floatValue());
      } else if (numericValue instanceof Long) {
        visitor.longField(info, numericValue.longValue());
      } else {
        throw new AssertionError();
      }
      continue;
    }
    // must be String
    if (f instanceof LargeLazyField) { // optimization to avoid premature string conversion
      visitor.stringField(info, toStringUnwrapIfPossible(((LargeLazyField) f).readBytes()));
    } else {
      visitor.stringField(info, f.stringValue());
    }
  }
}
 

private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException, IOException {
  reader.document(docId, new StoredFieldVisitor() {
    @Override
    public Status needsField(FieldInfo fieldInfo) throws IOException {
      if (ROW_ID.equals(fieldInfo.name)) {
        return StoredFieldVisitor.Status.STOP;
      }
      return StoredFieldVisitor.Status.NO;
    }
  });
  return reader.document(docId).get(ROW_ID);
}
 

public ComplianceAwareStoredFieldVisitor(final StoredFieldVisitor delegate) {
    super();
    this.delegate = delegate;
}
 

FilterStoredFieldVisitor(StoredFieldVisitor visitor) {
    this.visitor = visitor;
}
 

@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
    fieldsReader.visitDocument(docID, visitor);
}
 

public ReadMaskStoredFieldVisitor(StoredFieldVisitor visitor, Map<String, String> readMaskFieldsAndMessages) {
  _visitor = visitor;
  _readMaskFieldsAndMessages = readMaskFieldsAndMessages;
}
 

public void doc(int docID, StoredFieldVisitor fieldVisitor) throws IOException {
  _secureIndexReader.document(docID, fieldVisitor);
}
 

@Test
public void testEstimator() throws Exception {
  JettySolrRunner jetty = cluster.getRandomJetty(random());
  String randomCoreName = jetty.getCoreContainer().getAllCoreNames().iterator().next();
  SolrCore core = jetty.getCoreContainer().getCore(randomCoreName);
  RefCounted<SolrIndexSearcher> searcherRef = core.getSearcher();
  try {
    SolrIndexSearcher searcher = searcherRef.get();
    // limit the max length
    IndexSizeEstimator estimator = new IndexSizeEstimator(searcher.getRawReader(), 20, 50, true, true);
    IndexSizeEstimator.Estimate estimate = estimator.estimate();
    Map<String, Long> fieldsBySize = estimate.getFieldsBySize();
    assertFalse("empty fieldsBySize", fieldsBySize.isEmpty());
    assertEquals(fieldsBySize.toString(), fields.size(), fieldsBySize.size());
    fieldsBySize.forEach((k, v) -> assertTrue("unexpected size of " + k + ": " + v, v > 0));
    Map<String, Long> typesBySize = estimate.getTypesBySize();
    assertFalse("empty typesBySize", typesBySize.isEmpty());
    assertTrue("expected at least 8 types: " + typesBySize.toString(), typesBySize.size() >= 8);
    typesBySize.forEach((k, v) -> assertTrue("unexpected size of " + k + ": " + v, v > 0));
    Map<String, Object> summary = estimate.getSummary();
    assertNotNull("summary", summary);
    assertFalse("empty summary", summary.isEmpty());
    assertEquals(summary.keySet().toString(), fields.size(), summary.keySet().size());
    Map<String, Object> details = estimate.getDetails();
    assertNotNull("details", details);
    assertFalse("empty details", details.isEmpty());
    // by type
    assertEquals(details.keySet().toString(), 6, details.keySet().size());

    // check sampling
    estimator.setSamplingThreshold(searcher.getRawReader().maxDoc() / 2);
    IndexSizeEstimator.Estimate sampledEstimate = estimator.estimate();
    Map<String, Long> sampledFieldsBySize = sampledEstimate.getFieldsBySize();
    assertFalse("empty fieldsBySize", sampledFieldsBySize.isEmpty());
    // verify that the sampled values are within 50% of the original values
    fieldsBySize.forEach((field, size) -> {
      Long sampledSize = sampledFieldsBySize.get(field);
      assertNotNull("sampled size for " + field + " is missing in " + sampledFieldsBySize, sampledSize);
      double delta = (double) size * 0.5;
      assertEquals("sampled size of " + field + " is wildly off", (double)size, (double)sampledSize, delta);
    });
    // verify the reader is still usable - SOLR-13694
    IndexReader reader = searcher.getRawReader();
    for (LeafReaderContext context : reader.leaves()) {
      LeafReader leafReader = context.reader();
      assertTrue("unexpected LeafReader class: " + leafReader.getClass().getName(), leafReader instanceof CodecReader);
      Bits liveDocs = leafReader.getLiveDocs();
      CodecReader codecReader = (CodecReader) leafReader;
      StoredFieldsReader storedFieldsReader = codecReader.getFieldsReader();
      StoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
      assertNotNull(storedFieldsReader);
      for (int docId = 0; docId < leafReader.maxDoc(); docId++) {
        if (liveDocs != null && !liveDocs.get(docId)) {
          continue;
        }
        storedFieldsReader.visitDocument(docId, visitor);
      }
    }
  } finally {
    searcherRef.decref();
    core.close();
  }
}
 

/** Visit the stored fields for document <code>docID</code> */
public abstract void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException;
 

@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
  AssertingCodec.assertThread("StoredFieldsReader", creationThread);
  assert n >= 0 && n < maxDoc;
  in.visitDocument(n, visitor);
}
 

@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
  in.seek(offsets[n]);
  
  while (true) {
    readLine();
    if (StringHelper.startsWith(scratch.get(), FIELD) == false) {
      break;
    }
    int fieldNumber = parseIntAt(FIELD.length);
    FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
    readLine();
    assert StringHelper.startsWith(scratch.get(), NAME);
    readLine();
    assert StringHelper.startsWith(scratch.get(), TYPE);
    
    final BytesRef type;
    if (equalsAt(TYPE_STRING, scratch.get(), TYPE.length)) {
      type = TYPE_STRING;
    } else if (equalsAt(TYPE_BINARY, scratch.get(), TYPE.length)) {
      type = TYPE_BINARY;
    } else if (equalsAt(TYPE_INT, scratch.get(), TYPE.length)) {
      type = TYPE_INT;
    } else if (equalsAt(TYPE_LONG, scratch.get(), TYPE.length)) {
      type = TYPE_LONG;
    } else if (equalsAt(TYPE_FLOAT, scratch.get(), TYPE.length)) {
      type = TYPE_FLOAT;
    } else if (equalsAt(TYPE_DOUBLE, scratch.get(), TYPE.length)) {
      type = TYPE_DOUBLE;
    } else {
      throw new RuntimeException("unknown field type");
    }
    
    switch (visitor.needsField(fieldInfo)) {
      case YES:  
        readField(type, fieldInfo, visitor);
        break;
      case NO:   
        readLine();
        assert StringHelper.startsWith(scratch.get(), VALUE);
        break;
      case STOP: return;
    }
  }
}
 

@Override
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
}
 

public HashingStoredFieldVisitor(final StoredFieldVisitor delegate) {
    super();
    this.delegate = delegate;
}
 

public FlsStoredFieldVisitor(final StoredFieldVisitor delegate) {
    super();
    this.delegate = delegate;
}
 

/**
 * Visit a document's fields using a {@link StoredFieldVisitor}.
 * This method does not currently add to the Solr document cache.
 * 
 * @see IndexReader#document(int, StoredFieldVisitor)
 * @see SolrDocumentFetcher
 */
@Override
public final void doc(int docId, StoredFieldVisitor visitor) throws IOException {
  getDocFetcher().doc(docId, visitor);
}
 

/** 
 * Sugar for <code>.getIndexReader().document(docID, fieldVisitor)</code>
 * @see IndexReader#document(int, StoredFieldVisitor) 
 */
public void doc(int docID, StoredFieldVisitor fieldVisitor) throws IOException {
  reader.document(docID, fieldVisitor);
}