org.apache.distributedlog.exceptions.EndOfStreamException Java Examples

/**
 * Non durable write should fail if writer is marked as end of stream.
 *
 * @throws Exception
 */
@Test(timeout = 60000)
public void testNondurableWriteAfterEndOfStream() throws Exception {
    DistributedLogConfiguration confLocal = newLocalConf();
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setOutputBufferSize(Integer.MAX_VALUE);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setDurableWriteEnabled(false);
    ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true);
    BKLogSegmentWriter writer =
            createLogSegmentWriter(confLocal, 0L, -1L, lock);

    Utils.ioResult(writer.markEndOfStream());

    try {
        Utils.ioResult(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(1)));
        fail("Should fail the write if the writer is marked as end of stream");
    } catch (EndOfStreamException we) {
        // expected
    }

    closeWriterAndLock(writer, lock);
}
 

/**
 * Test Case: read records from the input stream.
 */
@Test
public void testRead() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

    byte[] data = "test-read".getBytes(UTF_8);
    LogRecordWithDLSN record = mock(LogRecordWithDLSN.class);
    when(record.getPayLoadInputStream())
        .thenReturn(new ByteArrayInputStream(data));
    when(reader.readNext(anyBoolean()))
        .thenReturn(record)
        .thenThrow(new EndOfStreamException("eos"));

    DLInputStream in = new DLInputStream(dlm);
    int numReads = 0;
    int readByte;
    while ((readByte = in.read()) != -1) {
        assertEquals(data[numReads], readByte);
        ++numReads;
    }
    assertEquals(data.length, numReads);
}
 

/**
 * Test Case: read records from the input stream.
 */
@Test
public void testRead() throws Exception {
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

  byte[] data = "test-read".getBytes(UTF_8);
  LogRecordWithDLSN record = mock(LogRecordWithDLSN.class);
  when(record.getPayLoadInputStream())
      .thenReturn(new ByteArrayInputStream(data));
  when(reader.readNext(anyBoolean()))
      .thenReturn(record)
      .thenThrow(new EndOfStreamException("eos"));

  DLInputStream in = new DLInputStream(dlm);
  int numReads = 0;
  int readByte;
  while ((readByte = in.read()) != -1) {
    assertEquals(data[numReads], readByte);
    ++numReads;
  }
  assertEquals(data.length, numReads);
}
 

@Test(timeout = 60000)
public void testWriteFailsAfterMarkEndOfStream() throws Exception {
    String name = "distrlog-mark-end-failure";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    txid = writeAndMarkEndOfStream(dlm, txid);

    assertEquals(txid - 1, dlm.getLastTxId());
    LogRecord last = dlm.getLastLogRecord();
    assertEquals(txid - 1, last.getTransactionId());
    DLMTestUtil.verifyLogRecord(last);
    assertTrue(dlm.isEndOfStreamMarked());

    LogWriter writer = null;
    boolean exceptionEncountered = false;
    try {
        writer = dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
            writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        }
    } catch (EndOfStreamException exc) {
        exceptionEncountered = true;
    }
    writer.close();
    assertTrue(exceptionEncountered);
}
 

/**
 * Get input stream representing next entry in the
 * ledger.
 *
 * @return input stream, or null if no more entries
 */
private LogRecordWithInputStream nextLogRecord() throws IOException {
  try {
    return nextLogRecord(reader);
  } catch (EndOfStreamException e) {
    eos = true;
    return null;
  }
}
 

/**
 * Test Case: reader hits eos (end of stream)
 */
@Test
public void testReadEos() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);
    when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));

    byte[] b = new byte[1];
    DLInputStream in = new DLInputStream(dlm);
    assertEquals("Should return 0 when reading an empty eos stream",
        0, in.read(b, 0, 1));
    assertEquals("Should return -1 when reading an empty eos stream",
        -1, in.read(b, 0, 1));
}
 

/**
 * Get input stream representing next entry in the
 * ledger.
 *
 * @return input stream, or null if no more entries
 */
private LogRecordWithInputStream nextLogRecord() throws IOException {
  try {
    return nextLogRecord(reader);
  } catch (EndOfStreamException e) {
    eos = true;
    LOG.info(()->"end of stream is reached");
    return null;
  }
}
 

/**
 * Test Case: reader hits eos (end of stream)
 */
@Test
public void testReadEos() throws Exception {
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);
  when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));

  byte[] b = new byte[1];
  DLInputStream in = new DLInputStream(dlm);
  assertEquals("Should return 0 when reading an empty eos stream",
      -1, in.read(b, 0, 1));
  assertEquals("Should return -1 when reading an empty eos stream",
      -1, in.read(b, 0, 1));
}
 

protected void doStartLogSegment(final long txId,
                                 final boolean bestEffort,
                                 final boolean allowMaxTxID,
                                 final CompletableFuture<BKLogSegmentWriter> promise) {
    // validate the tx id
    if ((txId < 0)
            || (!allowMaxTxID && (txId == DistributedLogConstants.MAX_TXID))) {
        FutureUtils.completeExceptionally(promise, new IOException("Invalid Transaction Id " + txId));
        return;
    }

    long highestTxIdWritten = maxTxId.get();
    if (txId < highestTxIdWritten) {
        if (highestTxIdWritten == DistributedLogConstants.MAX_TXID) {
            LOG.error("We've already marked the stream as ended and attempting to start a new log segment");
            FutureUtils.completeExceptionally(promise,
                    new EndOfStreamException("Writing to a stream after it has been marked as completed"));
            return;
        } else {
            LOG.error("We've already seen TxId {} the max TXId is {}", txId, highestTxIdWritten);
            FutureUtils.completeExceptionally(promise,
                    new TransactionIdOutOfOrderException(txId, highestTxIdWritten));
            return;
        }
    }

    try {
        logSegmentAllocator.allocate();
    } catch (IOException e) {
        // failed to issue an allocation request
        failStartLogSegment(promise, bestEffort, e);
        return;
    }

    // start the transaction from zookeeper
    final Transaction<Object> txn = streamMetadataStore.newTransaction();

    // failpoint injected before creating ledger
    try {
        FailpointUtils.checkFailPoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate);
    } catch (IOException ioe) {
        failStartLogSegment(promise, bestEffort, ioe);
        return;
    }

    logSegmentAllocator.tryObtain(txn, NULL_OP_LISTENER)
            .whenComplete(new FutureEventListener<LogSegmentEntryWriter>() {

        @Override
        public void onSuccess(LogSegmentEntryWriter entryWriter) {
            // try-obtain succeed
            createInprogressLogSegment(
                    txn,
                    txId,
                    entryWriter,
                    bestEffort,
                    promise);
        }

        @Override
        public void onFailure(Throwable cause) {
            failStartLogSegment(promise, bestEffort, cause);
        }
    });
}
 

private LogRecordWithDLSN doReadNext(boolean nonBlocking) throws IOException {
    LogRecordWithDLSN record = null;

    do {
        // fetch one record until we don't find any entry available in the readahead cache
        while (null == record) {
            if (null == currentEntry) {
                currentEntry = readNextEntry(nonBlocking);
                if (null == currentEntry) {
                    return null;
                }
            }
            record = currentEntry.nextRecord();
            if (null == record) {
                currentEntry = null;
            }
        }

        // check if we reached the end of stream
        if (record.isEndOfStream()) {
            EndOfStreamException eos = new EndOfStreamException("End of Stream Reached for "
                    + readHandler.getFullyQualifiedName());
            readerException.compareAndSet(null, eos);
            throw eos;
        }
        // skip control records
        if (record.isControl()) {
            record = null;
            continue;
        }
        if (!positioned) {
            if (record.getTransactionId() < startTransactionId.get()) {
                record = null;
                continue;
            } else {
                positioned = true;
                break;
            }
        } else {
            break;
        }
    } while (true);
    return record;
}
 

private synchronized CompletableFuture<DLSN> writeUserRecord(LogRecord record) throws IOException {
    if (null != closeFuture) {
        throw new WriteException(fullyQualifiedLogSegment,
                BKException.getMessage(BKException.Code.LedgerClosedException));
    }

    if (BKException.Code.OK != transmitResult.get()) {
        // Failfast if the stream already encountered error with safe retry on the client
        throw new WriteException(fullyQualifiedLogSegment, BKException.getMessage(transmitResult.get()));
    }

    if (streamEnded) {
        throw new EndOfStreamException("Writing to a stream after it has been marked as completed");
    }

    if ((record.getTransactionId() < 0)
            || (record.getTransactionId() == DistributedLogConstants.MAX_TXID)) {
        throw new TransactionIdOutOfOrderException(record.getTransactionId());
    }

    // Inject write delay if configured to do so
    writeDelayInjector.inject();

    // Will check write rate limits and throw if exceeded.
    writeLimiter.acquire();
    pendingWrites.inc();

    // The count represents the number of user records up to the
    // current record
    // Increment the record count only when writing a user log record
    // Internally generated log records don't increment the count
    // writeInternal will always set a count regardless of whether it was
    // incremented or not.
    CompletableFuture<DLSN> future = null;
    try {
        // increment the position for the record to write
        // if the record is failed to write, it would be decremented.
        positionWithinLogSegment++;
        int numRecords = 1;
        if (record.isRecordSet()) {
            numRecords = LogRecordSet.numRecords(record);
        }
        future = writeInternal(record);
        // after the record (record set) is written, the position should be
        // moved for {numRecords}, but since we already moved the record by 1
        // so advance the position for other {numRecords - 1}.
        positionWithinLogSegment += (numRecords - 1);
    } catch (IOException ex) {
        writeLimiter.release();
        pendingWrites.dec();
        positionWithinLogSegment--;
        throw ex;
    }

    // Track outstanding requests and return the future.
    return FutureUtils.ensure(future, () -> {
        pendingWrites.dec();
        writeLimiter.release();
    });
}
 

@Test
public void testDownload() throws Exception {
  String logName = "test-download";
  URI uri = URI.create("distributedlog://127.0.0.1/test/distributedlog/" + logName);

  File tempFile = File.createTempFile("test", "download");
  // make sure it is deleted when the test completes
  tempFile.deleteOnExit();
  Path path = Paths.get(tempFile.toURI());

  Namespace ns = mock(Namespace.class);
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(ns.openLog(anyString())).thenReturn(dlm);
  when(dlm.getInputStream(eq(DLSN.InitialDLSN))).thenReturn(reader);
  when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));

  NamespaceBuilder nsBuilder = mock(NamespaceBuilder.class);
  when(nsBuilder.clientId(anyString())).thenReturn(nsBuilder);
  when(nsBuilder.conf(any(DistributedLogConfiguration.class))).thenReturn(nsBuilder);
  when(nsBuilder.uri(any(URI.class))).thenReturn(nsBuilder);
  when(nsBuilder.build()).thenReturn(ns);

  PowerMockito.mockStatic(Extractor.class);
  PowerMockito.doNothing()
      .when(Extractor.class, "extract", any(InputStream.class), any(Path.class));

  DLDownloader downloader = new DLDownloader(() -> nsBuilder);
  downloader.download(uri, path);

  URI parentUri = URI.create("distributedlog://127.0.0.1/test/distributedlog");
  verify(nsBuilder, times(1)).clientId(eq("heron-downloader"));
  verify(nsBuilder, times(1)).conf(eq(CONF));
  verify(nsBuilder, times(1)).uri(parentUri);

  PowerMockito.verifyStatic(times(1));
  Extractor.extract(any(InputStream.class), eq(path));

  verify(ns, times(1)).openLog(eq(logName));
  verify(ns, times(1)).close();
}