org.apache.distributedlog.api.LogReader Java Examples

/**
 * Test Case: read records from the input stream.
 */
@Test
public void testRead() throws Exception {
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

  byte[] data = "test-read".getBytes(UTF_8);
  LogRecordWithDLSN record = mock(LogRecordWithDLSN.class);
  when(record.getPayLoadInputStream())
      .thenReturn(new ByteArrayInputStream(data));
  when(reader.readNext(anyBoolean()))
      .thenReturn(record)
      .thenThrow(new EndOfStreamException("eos"));

  DLInputStream in = new DLInputStream(dlm);
  int numReads = 0;
  int readByte;
  while ((readByte = in.read()) != -1) {
    assertEquals(data[numReads], readByte);
    ++numReads;
  }
  assertEquals(data.length, numReads);
}
 

@Test(timeout = 15000)
public void testHandleInconsistentMetadataNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < (numRecordsWritten / 2)) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}
 

@Test(timeout = 15000)
public void testHandleInconsistentMetadataDLSNNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read-dlsn";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < numRecordsWritten) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}
 

/**
 * Test Case: read records from the input stream.
 */
@Test
public void testRead() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

    byte[] data = "test-read".getBytes(UTF_8);
    LogRecordWithDLSN record = mock(LogRecordWithDLSN.class);
    when(record.getPayLoadInputStream())
        .thenReturn(new ByteArrayInputStream(data));
    when(reader.readNext(anyBoolean()))
        .thenReturn(record)
        .thenThrow(new EndOfStreamException("eos"));

    DLInputStream in = new DLInputStream(dlm);
    int numReads = 0;
    int readByte;
    while ((readByte = in.read()) != -1) {
        assertEquals(data[numReads], readByte);
        ++numReads;
    }
    assertEquals(data.length, numReads);
}
 

@Test(timeout = 60000)
public void testToolTruncateStream() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", confLocal, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 3, 1000);

    DLSN dlsn = new DLSN(2, 1, 0);
    TruncateStreamCommand cmd = new TruncateStreamCommand();
    cmd.setDlsn(dlsn);
    cmd.setUri(defaultUri);
    cmd.setStreamName("testToolTruncateStream");
    cmd.setForce(true);

    assertEquals(0, cmd.runCmd());

    LogReader reader = dlm.getInputStream(0);
    LogRecordWithDLSN record = reader.readNext(false);
    assertEquals(dlsn, record.getDlsn());

    reader.close();
    dlm.close();
}
 

private void verifyEntries(String name, long readFromTxId, long startTxId, int numEntries) throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, name);
    LogReader reader = dlm.getInputStream(readFromTxId);

    long txid = startTxId;
    int numRead = 0;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        DLMTestUtil.verifyLogRecord(r);
        assertEquals(txid++, r.getTransactionId());
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(numEntries, numRead);
    reader.close();
    dlm.close();
}
 

@Override
public FSDataInputStream open(Path path, int bufferSize)
        throws IOException {
    try {
        DistributedLogManager dlm = namespace.openLog(getStreamName(path));
        LogReader reader;
        try {
            reader = dlm.openLogReader(DLSN.InitialDLSN);
        } catch (LogNotFoundException lnfe) {
            throw new FileNotFoundException(path.toString());
        } catch (LogEmptyException lee) {
            throw new FileNotFoundException(path.toString());
        }
        return new FSDataInputStream(
            new BufferedFSInputStream(
                new DLInputStream(dlm, reader, 0L),
                bufferSize));
    } catch (LogNotFoundException e) {
        throw new FileNotFoundException(path.toString());
    }
}
 

private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException {
    LogRecordWithDLSN record = reader.readNext(false);

    if (null != record) {
        return new LogRecordWithInputStream(record);
    } else {
        record = reader.readNext(false);
        if (null != record) {
            return new LogRecordWithInputStream(record);
        } else {
            LOG.debug("No record");
            return null;
        }
    }
}
 

@Test(timeout = 60000)
public void testReadingFromEmptyLog() throws Exception {
    String name = testName.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    // write a record but not commit
    LogRecord op = DLMTestUtil.getLogRecordInstance(1L);
    out.write(op);

    LogReader reader = dlm.getInputStream(1L);
    assertNull(reader.readNext(true));
    assertNull(reader.readNext(false));

    op = DLMTestUtil.getLogRecordInstance(2L);
    out.write(op);

    // reader is able to read first record
    LogRecord record = waitForNextRecord(reader);
    assertNotNull(record);
    assertEquals(1L, record.getTransactionId());
    DLMTestUtil.verifyLogRecord(record);

    assertNull(reader.readNext(true));

    out.close();
    reader.close();
    dlm.close();
}
 

@Test(timeout = 60000)
public void testHandleInconsistentMetadata() throws Exception {
    String name = "distrlog-inconsistent-metadata-blocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        LogRecord record = reader.readNext(false);
        long lastTxId = -1;
        while (numRecordsRead < numRecordsWritten / 2) {
            if (null != record) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
            record = reader.readNext(false);
        }
        reader.close();
        assertEquals(numRecordsWritten / 2, numRecordsRead);
    } finally {
        dlm.close();
    }
}
 

private static RecordStream nextRecordStream(LogReader reader) throws IOException {
    LogRecordWithDLSN record = reader.readNext(false);
    if (null != record) {
        return new RecordStream(record);
    }
    return null;
}
 

DLInputStream(DistributedLogManager dlm,
              LogReader reader,
              long startPos)
        throws IOException {
    this.dlm = dlm;
    this.reader = reader;
    this.pos = startPos;
    this.lastPos = readEndPos();
    seek(startPos);
}
 

@Test
public void testGetInputStream() throws Exception {
    LogReader reader = mock(LogReader.class);
    when(impl.getInputStream(anyLong())).thenReturn(reader);
    assertEquals(reader, ((LogReaderImpl) manager.getInputStream(1234L)).getImpl());
    verify(impl, times(1)).getInputStream(eq(1234L));
}
 

@Test
public void testGetInputStream2() throws Exception {
    DLSN dlsn = mock(DLSN.class);
    LogReader reader = mock(LogReader.class);
    when(impl.getInputStream(eq(dlsn))).thenReturn(reader);
    assertEquals(reader, ((LogReaderImpl) manager.getInputStream(dlsn)).getImpl());
    verify(impl, times(1)).getInputStream(eq(dlsn));
}
 

private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException {
  LogRecordWithDLSN record = reader.readNext(false);

  if (null != record) {
    return new LogRecordWithInputStream(record);
  } else {
    record = reader.readNext(false);
    if (null != record) {
      return new LogRecordWithInputStream(record);
    } else {
      return null;
    }
  }
}
 

/**
 * Test Case: reader hits eos (end of stream)
 */
@Test
public void testReadEos() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);
    when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));

    byte[] b = new byte[1];
    DLInputStream in = new DLInputStream(dlm);
    assertEquals("Should return 0 when reading an empty eos stream",
        0, in.read(b, 0, 1));
    assertEquals("Should return -1 when reading an empty eos stream",
        -1, in.read(b, 0, 1));
}
 

/**
 * Test Case: close the input stream
 */
@Test
public void testClose() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

    DLInputStream in = new DLInputStream(dlm);
    verify(dlm, times(1)).getInputStream(eq(DLSN.InitialDLSN));
    in.close();
    verify(dlm, times(1)).close();
    verify(reader, times(1)).close();
}
 

@Test
public void testOpenInputStream() throws Exception {
  Namespace ns = mock(Namespace.class);
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(ns.openLog(anyString())).thenReturn(dlm);
  when(dlm.getInputStream(eq(DLSN.InitialDLSN))).thenReturn(reader);

  InputStream is = DLDownloader.openInputStream(ns, "test-open-inputstream");
  assertTrue(is instanceof DLInputStream);

  is.close();
  verify(dlm, times(1)).close();
  verify(reader, times(1)).close();
}
 

private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException {
  LogRecordWithDLSN record = reader.readNext(false);

  if (null != record) {
    return new LogRecordWithInputStream(record);
  } else {
    record = reader.readNext(false);
    if (null != record) {
      return new LogRecordWithInputStream(record);
    } else {
      return null;
    }
  }
}
 

/**
 * Test Case: reader hits eos (end of stream)
 */
@Test
public void testReadEos() throws Exception {
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);
  when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));

  byte[] b = new byte[1];
  DLInputStream in = new DLInputStream(dlm);
  assertEquals("Should return 0 when reading an empty eos stream",
      -1, in.read(b, 0, 1));
  assertEquals("Should return -1 when reading an empty eos stream",
      -1, in.read(b, 0, 1));
}
 

/**
 * Test Case: close the input stream
 */
@Test
public void testClose() throws Exception {
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

  DLInputStream in = new DLInputStream(dlm);
  verify(dlm, times(1)).getInputStream(eq(DLSN.InitialDLSN));
  in.close();
  verify(dlm, times(1)).close();
  verify(reader, times(1)).close();
}
 

static long getNumberofLogRecords(DistributedLogManager bkdlm, long startTxId) throws IOException {
    long numLogRecs = 0;
    LogReader reader = bkdlm.getInputStream(startTxId);
    LogRecord record = reader.readNext(false);
    while (null != record) {
        numLogRecs++;
        verifyLogRecord(record);
        record = reader.readNext(false);
    }
    reader.close();
    return numLogRecs;
}
 

int countFromStartToEnd(DistributedLogManager dlm) throws Exception {
    int count = 0;
    try {
        LogReader reader = dlm.getInputStream(startDLSN);
        try {
            LogRecordWithDLSN record = reader.readNext(false);
            LogRecordWithDLSN preRecord = record;
            System.out.println("first record : " + record);
            while (null != record) {
                if (record.getDlsn().compareTo(endDLSN) > 0) {
                    break;
                }
                ++count;
                if (count % 1000 == 0) {
                    logger.info("read {} records from {}...", count, getStreamName());
                }
                preRecord = record;
                record = reader.readNext(false);
            }
            System.out.println("last record : " + preRecord);
        } finally {
            reader.close();
        }
    } finally {
        dlm.close();
    }
    return count;
}
 

/**
 * Get the input stream starting with fromTxnId for the specified log.
 *
 * @param fromTxnId
 *          transaction id to start reading from
 * @return log reader
 * @throws IOException
 */
LogReader getInputStreamInternal(long fromTxnId)
    throws IOException {
    DLSN fromDLSN;
    try {
        fromDLSN = Utils.ioResult(getDLSNNotLessThanTxId(fromTxnId));
    } catch (LogEmptyException lee) {
        fromDLSN = DLSN.InitialDLSN;
    }
    return getInputStreamInternal(fromDLSN, Optional.of(fromTxnId));
}
 

LogReader getInputStreamInternal(DLSN fromDLSN, Optional<Long> fromTxnId)
        throws IOException {
    LOG.info("Create sync reader starting from {}", fromDLSN);
    checkClosedOrInError("getInputStream");
    return new BKSyncLogReader(
            conf,
            this,
            fromDLSN,
            fromTxnId,
            statsLogger);
}
 

private int drainStreams(LogReader reader0, int num0, LogReader reader1, int num1)
        throws Exception {
    // Allow time for watches to fire
    Thread.sleep(15);
    int numTrans = 0;
    LogRecord record;
    int i = 0;
    while (i < num0) {
        record = reader0.readNext(false);
        if (null != record) {
            assertTrue((record.getTransactionId() % 2 == 0));
            DLMTestUtil.verifyLogRecord(record);
            numTrans++;
            i++;
            LOG.info("Read record {}", record);
        }
    }
    i = 0;
    while (i < num1) {
        record = reader1.readNext(false);
        if (null != record) {
            assertTrue((record.getTransactionId() % 2 == 1));
            DLMTestUtil.verifyLogRecord(record);
            numTrans++;
            i++;
            LOG.info("Read record {}", record);
        }
    }
    return numTrans;
}
 

@Test(timeout = 60000)
public void testInterleavedReaders() throws Exception {
    String name = "distrlog-interleaved";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    BKDistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");

    LogReader reader0 = null;
    LogReader reader1 = null;
    long txid = 1;
    int numTrans = 0;

    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 4; j++) {
        for (int k = 1; k <= 10; k++) {
            Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        if (null == reader0) {
            reader0 = dlmreader0.getInputStream(1);
        }
        if (null == reader1) {
            reader1 = dlmreader1.getInputStream(1);
        }
        numTrans += drainStreams(reader0, 10, reader1, 10);
        assertEquals((txid - 1), numTrans);
    }
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}
 

protected LogRecord waitForNextRecord(LogReader reader) throws Exception {
    LogRecord record = reader.readNext(false);
    while (null == record) {
        record = reader.readNext(false);
    }
    return record;
}
 

@Test(timeout = 60000)
public void testInterleavedReadersWithCleanup() throws Exception {
    String name = "distrlog-interleaved-cleanup";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    long txid = 1;
    Long retentionPeriodOverride = null;

    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 4; j++) {
        for (int k = 1; k <= 10; k++) {
            if (k == 5) {
                writer0.setForceRolling(true);
                writer0.overRideMinTimeStampToKeep(retentionPeriodOverride);
                writer1.setForceRolling(true);
                writer1.overRideMinTimeStampToKeep(retentionPeriodOverride);
            }
            DLSN dlsn1 = Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LOG.info("writer1 write record {}", dlsn1);
            DLSN dlsn0 = Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LOG.info("writer0 write record {}", dlsn0);
            if (k == 5) {
                writer0.setForceRolling(false);
                writer1.setForceRolling(false);
                retentionPeriodOverride = System.currentTimeMillis();
            }
            Thread.sleep(5);
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
    }
    writer0.close();
    writer1.close();

    DistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    DistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");
    LogReader reader0 = dlmreader0.getInputStream(1);
    LogReader reader1 = dlmreader1.getInputStream(1);
    int numTrans = drainStreams(reader0, 15, reader1, 15);
    assertEquals(30, numTrans);
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}
 

@Test(timeout = 60000)
public void testInterleavedReadersWithRolling() throws Exception {
    String name = "distrlog-interleaved-rolling";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    BKDistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");

    LogReader reader0 = null;
    LogReader reader1 = null;
    long txid = 1;
    int numTrans = 0;

    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 2; j++) {
        for (int k = 1; k <= 6; k++) {
            if (k == 3) {
                writer0.setForceRolling(true);
                writer1.setForceRolling(true);
            }
            Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
            writer0.setForceRolling(false);
            writer1.setForceRolling(false);
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        if (null == reader0) {
            reader0 = dlmreader0.getInputStream(1);
        }
        if (null == reader1) {
            reader1 = dlmreader1.getInputStream(1);
        }
        numTrans += drainStreams(reader0, 6, reader1, 6);
        assertEquals((txid - 1), numTrans);
    }
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}