Java Code Examples for org.apache.lucene.util.IOUtils#rethrowAlways()

@Override
public void mergeFinished(boolean success, boolean segmentDropped) throws IOException {
  super.mergeFinished(success, segmentDropped);
  Throwable th = null;
  for (ParallelLeafReader r : parallelReaders) {
    try {
      r.decRef();
    } catch (Throwable t) {
      if (th == null) {
        th = t;
      }
    }
  }

  if (th != null) {
    throw IOUtils.rethrowAlways(th);
  }
}
 

private static void checkSoftDeletes(String softDeletesField, SegmentCommitInfo info, SegmentReader reader, PrintStream infoStream, boolean failFast) throws IOException {
  if (infoStream != null)
    infoStream.print("    test: check soft deletes.....");
  try {
    int softDeletes = PendingSoftDeletes.countSoftDeletes(DocValuesFieldExistsQuery.getDocValuesDocIdSetIterator(softDeletesField, reader), reader.getLiveDocs());
    if (softDeletes != info.getSoftDelCount()) {
      throw new RuntimeException("actual soft deletes: " + softDeletes + " but expected: " +info.getSoftDelCount());
    }
  } catch (Exception e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }
}
 

@Override
public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
  Throwable exception = null;
  try {
    return super.newFileChannel(path, options, attrs);
  } catch (Throwable t) {
    exception = t;
  } finally {
    if (containsDestructive(options)) {
      sop("newFileChannel" + options + ": " + path(path), exception);
    } else {
      if (exception != null) {
        throw IOUtils.rethrowAlways(exception);
      }
    }
  }
  throw new AssertionError();
}
 

@Override
public AsynchronousFileChannel newAsynchronousFileChannel(Path path, Set<? extends OpenOption> options, ExecutorService executor, FileAttribute<?>... attrs) throws IOException {
  Throwable exception = null;
  try {
    return super.newAsynchronousFileChannel(path, options, executor, attrs);
  } catch (Throwable t) {
    exception = t;
  } finally {
    if (containsDestructive(options)) {
      sop("newAsynchronousFileChannel" + options + ": " + path(path), exception);
    } else {
      if (exception != null) {
        throw IOUtils.rethrowAlways(exception);
      }
    }
  }
  throw new AssertionError();
}
 

@Override
public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
  Throwable exception = null;
  try {
    return super.newByteChannel(path, options, attrs);
  } catch (Throwable t) {
    exception = t;
  } finally {
    if (containsDestructive(options)) {
      sop("newByteChannel" + options + ": " + path(path), exception);
    } else {
      if (exception != null) {
        throw IOUtils.rethrowAlways(exception);
      }
    }
  }
  throw new AssertionError();
}
 

/**
 * Iterate through the failures list, giving each object a
 * chance to throw an IOE
 */
synchronized void maybeThrowDeterministicException() throws IOException {
  if (failures != null) {
    for(int i = 0; i < failures.size(); i++) {
      try {
        failures.get(i).eval(this);
      } catch (Throwable t) {
        if (LuceneTestCase.VERBOSE) {
          System.out.println("MockDirectoryWrapper: throw exc");
          t.printStackTrace(System.out);
        }
        throw IOUtils.rethrowAlways(t);
      }
    }
  }
}
 

/**
 * Do a specific action and validate after the action that the status is still OK, 
 * and if not, attempt to extract the actual server side exception. Optionally
 * release the response at exit, depending on <code>consume</code> parameter.
 */
protected <T> T doAction(HttpResponse response, boolean consume, Callable<T> call) throws IOException {
  Throwable th = null;
  try {
    return call.call();
  } catch (Throwable t) {
    th = t;
  } finally {
    try {
      verifyStatus(response);
    } finally {
      if (consume) {
        EntityUtils.consumeQuietly(response.getEntity());
      }
    }
  }
  throw IOUtils.rethrowAlways(th); 
}
 

/** 
 * Validates the codec footer previously written by {@link #writeFooter}, optionally
 * passing an unexpected exception that has already occurred.
 * <p>
 * When a {@code priorException} is provided, this method will add a suppressed exception 
 * indicating whether the checksum for the stream passes, fails, or cannot be computed, and 
 * rethrow it. Otherwise it behaves the same as {@link #checkFooter(ChecksumIndexInput)}.
 * <p>
 * Example usage:
 * <pre class="prettyprint">
 * try (ChecksumIndexInput input = ...) {
 *   Throwable priorE = null;
 *   try {
 *     // ... read a bunch of stuff ... 
 *   } catch (Throwable exception) {
 *     priorE = exception;
 *   } finally {
 *     CodecUtil.checkFooter(input, priorE);
 *   }
 * }
 * </pre>
 */
public static void checkFooter(ChecksumIndexInput in, Throwable priorException) throws IOException {
  if (priorException == null) {
    checkFooter(in);
  } else {
    try {
      // If we have evidence of corruption then we return the corruption as the
      // main exception and the prior exception gets suppressed. Otherwise we
      // return the prior exception with a suppressed exception that notifies
      // the user that checksums matched.
      long remaining = in.length() - in.getFilePointer();
      if (remaining < footerLength()) {
        // corruption caused us to read into the checksum footer already: we can't proceed
        throw new CorruptIndexException("checksum status indeterminate: remaining=" + remaining +
                                        "; please run checkindex for more details", in);
      } else {
        // otherwise, skip any unread bytes.
        in.skipBytes(remaining - footerLength());
        
        // now check the footer
        long checksum = checkFooter(in);
        priorException.addSuppressed(new CorruptIndexException("checksum passed (" + Long.toHexString(checksum) +
                                                               "). possibly transient resource issue, or a Lucene or JVM bug", in));
      }
    } catch (CorruptIndexException corruptException) {
      corruptException.addSuppressed(priorException);
      throw corruptException;
    } catch (Throwable t) {
      // catch-all for things that shouldn't go wrong (e.g. OOM during readInt) but could...
      priorException.addSuppressed(new CorruptIndexException("checksum status indeterminate: unexpected exception", in, t));
    }
    throw IOUtils.rethrowAlways(priorException);
  }
}
 

/**
 * Test the term index.
 * @lucene.experimental
 */
public static Status.TermIndexStatus testPostings(CodecReader reader, PrintStream infoStream, boolean verbose, boolean doSlowChecks, boolean failFast) throws IOException {

  // TODO: we should go and verify term vectors match, if
  // doSlowChecks is on...

  Status.TermIndexStatus status;
  final int maxDoc = reader.maxDoc();

  try {
    if (infoStream != null) {
      infoStream.print("    test: terms, freq, prox...");
    }

    final Fields fields = reader.getPostingsReader().getMergeInstance();
    final FieldInfos fieldInfos = reader.getFieldInfos();
    NormsProducer normsProducer = reader.getNormsReader();
    if (normsProducer != null) {
      normsProducer = normsProducer.getMergeInstance();
    }
    status = checkFields(fields, reader.getLiveDocs(), maxDoc, fieldInfos, normsProducer, true, false, infoStream, verbose, doSlowChecks);
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR: " + e);
    status = new Status.TermIndexStatus();
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }

  return status;
}
 

/**
 * Test field norms.
 * @lucene.experimental
 */
public static Status.FieldNormStatus testFieldNorms(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
  long startNS = System.nanoTime();
  final Status.FieldNormStatus status = new Status.FieldNormStatus();

  try {
    // Test Field Norms
    if (infoStream != null) {
      infoStream.print("    test: field norms.........");
    }
    NormsProducer normsReader = reader.getNormsReader();
    if (normsReader != null) {
      normsReader = normsReader.getMergeInstance();
    }
    for (FieldInfo info : reader.getFieldInfos()) {
      if (info.hasNorms()) {
        checkNumericDocValues(info.name, normsReader.getNorms(info), normsReader.getNorms(info));
        ++status.totFields;
      }
    }

    msg(infoStream, String.format(Locale.ROOT, "OK [%d fields] [took %.3f sec]", status.totFields, nsToSec(System.nanoTime()-startNS)));
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }

  return status;
}
 

/**
 * Test field infos.
 * @lucene.experimental
 */
public static Status.FieldInfoStatus testFieldInfos(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
  long startNS = System.nanoTime();
  final Status.FieldInfoStatus status = new Status.FieldInfoStatus();
  
  try {
    // Test Field Infos
    if (infoStream != null) {
      infoStream.print("    test: field infos.........");
    }
    FieldInfos fieldInfos = reader.getFieldInfos();
    for (FieldInfo f : fieldInfos) {
      f.checkConsistency();
    }
    msg(infoStream, String.format(Locale.ROOT, "OK [%d fields] [took %.3f sec]", fieldInfos.size(), nsToSec(System.nanoTime()-startNS)));
    status.totFields = fieldInfos.size();
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }
  
  return status;
}
 

/** Records message, and rethrows exception if not null */
private void sop(String text, Throwable exception) throws IOException {
  if (exception == null) {
    if (infoStream.isEnabled("FS")) {
      infoStream.message("FS", text);
    }
  } else {
    if (infoStream.isEnabled("FS")) {
      infoStream.message("FS", text + " (FAILED: " + exception + ")");
    }
    throw IOUtils.rethrowAlways(exception);
  }
}
 

/** Does all the "real work" of tallying up the counts. */
private final void countAll() throws IOException, InterruptedException {
  //System.out.println("ssdv count");

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  List<Future<Void>> results = new ArrayList<>();

  for (LeafReaderContext context : state.getReader().leaves()) {
    results.add(exec.submit(new CountOneSegment(context.reader(), null, ordinalMap, context.ord)));
  }

  for (Future<Void> result : results) {
    try {
      result.get();
    } catch (ExecutionException ee) {
      // Theoretically cause can be null; guard against that.
      Throwable cause = ee.getCause();
      throw IOUtils.rethrowAlways(cause != null ? cause : ee);
    }
  }
}
 

/** Does all the "real work" of tallying up the counts. */
private final void count(List<MatchingDocs> matchingDocs) throws IOException, InterruptedException {

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues && matchingDocs.size() > 1) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  IndexReader reader = state.getReader();
  List<Future<Void>> results = new ArrayList<>();

  for (MatchingDocs hits : matchingDocs) {
    // LUCENE-5090: make sure the provided reader context "matches"
    // the top-level reader passed to the
    // SortedSetDocValuesReaderState, else cryptic
    // AIOOBE can happen:
    if (ReaderUtil.getTopLevelContext(hits.context).reader() != reader) {
      throw new IllegalStateException("the SortedSetDocValuesReaderState provided to this class does not match the reader being searched; you must create a new SortedSetDocValuesReaderState every time you open a new IndexReader");
    }
    
    results.add(exec.submit(new CountOneSegment(hits.context.reader(), hits, ordinalMap, hits.context.ord)));
  }

  for (Future<Void> result : results) {
    try {
      result.get();
    } catch (ExecutionException ee) {
      // Theoretically cause can be null; guard against that.
      Throwable cause = ee.getCause();
      throw IOUtils.rethrowAlways(cause != null ? cause : ee);
    }
  }
}
 

/** Transfers whatever tmp files were already copied in this previous job and cancels the previous job */
public synchronized void transferAndCancel(CopyJob prevJob) throws IOException {
  synchronized(prevJob) {
    dest.message("CopyJob: now transfer prevJob " + prevJob);
    try {
      _transferAndCancel(prevJob);
    } catch (Throwable t) {
      dest.message("xfer: exc during transferAndCancel");
      cancel("exc during transferAndCancel", t);
      throw IOUtils.rethrowAlways(t);
    }
  }
}
 

/**
 * Tests index sort order.
 * @lucene.experimental
 */
public static Status.IndexSortStatus testSort(CodecReader reader, Sort sort, PrintStream infoStream, boolean failFast) throws IOException {
  // This segment claims its documents are sorted according to the incoming sort ... let's make sure:

  long startNS = System.nanoTime();

  Status.IndexSortStatus status = new Status.IndexSortStatus();

  if (sort != null) {
    if (infoStream != null) {
      infoStream.print("    test: index sort..........");
    }

    SortField fields[] = sort.getSort();
    final int reverseMul[] = new int[fields.length];
    final LeafFieldComparator comparators[] = new LeafFieldComparator[fields.length];

    LeafReaderContext readerContext = new LeafReaderContext(reader);
  
    for (int i = 0; i < fields.length; i++) {
      reverseMul[i] = fields[i].getReverse() ? -1 : 1;
      comparators[i] = fields[i].getComparator(1, i).getLeafComparator(readerContext);
    }

    int maxDoc = reader.maxDoc();

    try {

      for(int docID=1;docID < maxDoc;docID++) {
    
        int cmp = 0;

        for (int i = 0; i < comparators.length; i++) {
          // TODO: would be better if copy() didnt cause a term lookup in TermOrdVal & co,
          // the segments are always the same here...
          comparators[i].copy(0, docID-1);
          comparators[i].setBottom(0);
          cmp = reverseMul[i] * comparators[i].compareBottom(docID);
          if (cmp != 0) {
            break;
          }
        }

        if (cmp > 0) {
          throw new RuntimeException("segment has indexSort=" + sort + " but docID=" + (docID-1) + " sorts after docID=" + docID);
        }
      }
      msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", nsToSec(System.nanoTime()-startNS)));
    } catch (Throwable e) {
      if (failFast) {
        throw IOUtils.rethrowAlways(e);
      }
      msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
      status.error = e;
      if (infoStream != null) {
        e.printStackTrace(infoStream);
      }
    }
  }

  return status;
}
 

/**
 * Test the points index
 * @lucene.experimental
 */
public static Status.PointsStatus testPoints(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
  if (infoStream != null) {
    infoStream.print("    test: points..............");
  }
  long startNS = System.nanoTime();
  FieldInfos fieldInfos = reader.getFieldInfos();
  Status.PointsStatus status = new Status.PointsStatus();
  try {

    if (fieldInfos.hasPointValues()) {
      PointsReader pointsReader = reader.getPointsReader();
      if (pointsReader == null) {
        throw new RuntimeException("there are fields with points, but reader.getPointsReader() is null");
      }
      for (FieldInfo fieldInfo : fieldInfos) {
        if (fieldInfo.getPointDimensionCount() > 0) {
          PointValues values = pointsReader.getValues(fieldInfo.name);
          if (values == null) {
            continue;
          }

          status.totalValueFields++;

          long size = values.size();
          int docCount = values.getDocCount();

          final long crossCost = values.estimatePointCount(new ConstantRelationIntersectVisitor(Relation.CELL_CROSSES_QUERY));
          if (crossCost < size / 2) {
            throw new RuntimeException("estimatePointCount should return >= size/2 when all cells match");
          }
          final long insideCost = values.estimatePointCount(new ConstantRelationIntersectVisitor(Relation.CELL_INSIDE_QUERY));
          if (insideCost < size) {
            throw new RuntimeException("estimatePointCount should return >= size when all cells fully match");
          }
          final long outsideCost = values.estimatePointCount(new ConstantRelationIntersectVisitor(Relation.CELL_OUTSIDE_QUERY));
          if (outsideCost != 0) {
            throw new RuntimeException("estimatePointCount should return 0 when no cells match");
          }

          VerifyPointsVisitor visitor = new VerifyPointsVisitor(fieldInfo.name, reader.maxDoc(), values);
          values.intersect(visitor);

          if (visitor.getPointCountSeen() != size) {
            throw new RuntimeException("point values for field \"" + fieldInfo.name + "\" claims to have size=" + size + " points, but in fact has " + visitor.getPointCountSeen());
          }

          if (visitor.getDocCountSeen() != docCount) {
            throw new RuntimeException("point values for field \"" + fieldInfo.name + "\" claims to have docCount=" + docCount + " but in fact has " + visitor.getDocCountSeen());
          }

          status.totalValuePoints += visitor.getPointCountSeen();
        }
      }
    }

    msg(infoStream, String.format(Locale.ROOT, "OK [%d fields, %d points] [took %.3f sec]", status.totalValueFields, status.totalValuePoints, nsToSec(System.nanoTime()-startNS)));

  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR: " + e);
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }

  return status;
}
 

/**
 * Test stored fields.
 * @lucene.experimental
 */
public static Status.StoredFieldStatus testStoredFields(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
  long startNS = System.nanoTime();
  final Status.StoredFieldStatus status = new Status.StoredFieldStatus();

  try {
    if (infoStream != null) {
      infoStream.print("    test: stored fields.......");
    }

    // Scan stored fields for all documents
    final Bits liveDocs = reader.getLiveDocs();
    StoredFieldsReader storedFields = reader.getFieldsReader().getMergeInstance();
    for (int j = 0; j < reader.maxDoc(); ++j) {
      // Intentionally pull even deleted documents to
      // make sure they too are not corrupt:
      DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
      storedFields.visitDocument(j, visitor);
      Document doc = visitor.getDocument();
      if (liveDocs == null || liveDocs.get(j)) {
        status.docCount++;
        status.totFields += doc.getFields().size();
      }
    }      

    // Validate docCount
    if (status.docCount != reader.numDocs()) {
      throw new RuntimeException("docCount=" + status.docCount + " but saw " + status.docCount + " undeleted docs");
    }

    msg(infoStream, String.format(Locale.ROOT, "OK [%d total field count; avg %.1f fields per doc] [took %.3f sec]",
                                  status.totFields,
                                  (((float) status.totFields)/status.docCount),
                                  nsToSec(System.nanoTime() - startNS)));
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }

  return status;
}
 

/**
 * Test docvalues.
 * @lucene.experimental
 */
public static Status.DocValuesStatus testDocValues(CodecReader reader,
                                                   PrintStream infoStream,
                                                   boolean failFast) throws IOException {
  long startNS = System.nanoTime();
  final Status.DocValuesStatus status = new Status.DocValuesStatus();
  try {
    if (infoStream != null) {
      infoStream.print("    test: docvalues...........");
    }
    DocValuesProducer dvReader = reader.getDocValuesReader();
    if (dvReader != null) {
      dvReader = dvReader.getMergeInstance();
    }
    for (FieldInfo fieldInfo : reader.getFieldInfos()) {
      if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
        status.totalValueFields++;
        checkDocValues(fieldInfo, dvReader, reader.maxDoc(), infoStream, status);
      }
    }

    msg(infoStream, String.format(Locale.ROOT,
                                  "OK [%d docvalues fields; %d BINARY; %d NUMERIC; %d SORTED; %d SORTED_NUMERIC; %d SORTED_SET] [took %.3f sec]",
                                  status.totalValueFields,
                                  status.totalBinaryFields,
                                  status.totalNumericFields,
                                  status.totalSortedFields,
                                  status.totalSortedNumericFields,
                                  status.totalSortedSetFields,
                                  nsToSec(System.nanoTime()-startNS)));
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }
  return status;
}