Java Code Examples for org.apache.distributedlog.LogRecordWithDLSN#getDlsn()

public void processRecord(final LogRecordWithDLSN record) {
    Message msg;
    try {
        msg = Utils.parseMessage(record.getPayload());
    } catch (TException e) {
        invalidRecordsCounter.inc();
        LOG.warn("Failed to parse record {} for stream {} : size = {} , ",
                 new Object[] { record, streamIdx, record.getPayload().length, e });
        return;
    }
    long curTimeMillis = System.currentTimeMillis();
    long e2eLatency = curTimeMillis - msg.getPublishTime();
    long deliveryLatency = curTimeMillis - record.getTransactionId();
    if (e2eLatency >= 0) {
        e2eStat.registerSuccessfulEvent(e2eLatency, TimeUnit.MILLISECONDS);
    } else {
        negativeE2EStat.registerSuccessfulEvent(-e2eLatency, TimeUnit.MILLISECONDS);
    }
    if (deliveryLatency >= 0) {
        deliveryStat.registerSuccessfulEvent(deliveryLatency, TimeUnit.MILLISECONDS);
    } else {
        negativeDeliveryStat.registerSuccessfulEvent(-deliveryLatency, TimeUnit.MILLISECONDS);
    }

    prevDLSN = record.getDlsn();
}
 

@Override
public CompletableFuture<LogSegmentMetadata> updateLastRecord(LogSegmentMetadata segment,
                                                              LogRecordWithDLSN record) {
    DLSN dlsn = record.getDlsn();
    checkState(!segment.isInProgress(),
            "Updating last dlsn for an inprogress log segment isn't supported.");
    checkArgument(segment.isDLSNinThisSegment(dlsn),
            "DLSN " + dlsn + " doesn't belong to segment " + segment);
    final LogSegmentMetadata newSegment = segment.mutator()
            .setLastDLSN(dlsn)
            .setLastTxId(record.getTransactionId())
            .setRecordCount(record)
            .build();
    return updateSegmentMetadata(newSegment);
}
 

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }

    String dlUriStr = args[0];
    final String streamName = args[1];

    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();

    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);

    // get the last record
    LogRecordWithDLSN lastRecord;
    DLSN dlsn;
    try {
        lastRecord = dlm.getLastLogRecord();
        dlsn = lastRecord.getDlsn();
        readLoop(dlm, dlsn);
    } catch (LogNotFoundException lnfe) {
        System.err.println("Log stream " + streamName + " is not found. Please create it first.");
        return;
    } catch (LogEmptyException lee) {
        System.err.println("Log stream " + streamName + " is empty.");
        dlsn = DLSN.InitialDLSN;
        readLoop(dlm, dlsn);
    } finally {
        dlm.close();
        namespace.close();
    }
}
 

private static void transform(final AsyncLogWriter writer,
                              LogRecordWithDLSN record,
                              Transformer<byte[], byte[]> replicationTransformer,
                              final CountDownLatch keepAliveLatch)
        throws Exception {
    DLSN srcDLSN = record.getDlsn();
    byte[] payload = record.getPayload();
    byte[] transformedPayload = replicationTransformer.transform(payload);
    TransformedRecord transformedRecord =
            new TransformedRecord(ByteBuffer.wrap(transformedPayload));
    transformedRecord.setSrcDlsn(srcDLSN.serializeBytes());
    ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
    transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos)));
    byte[] data = baos.toByteArray();
    writer.write(new LogRecord(record.getSequenceId(), data))
            .whenComplete(new FutureEventListener<DLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on writing records to stream " + writer.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            System.out.println("Write transformed record " + dlsn);
        }
    });
}
 

@Test(timeout = 60000)
public void testReadEntriesFromCompleteLogSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    boolean done = false;
    long txId = 1L;
    long entryId = 0L;
    while (!done) {
        Entry.Reader entryReader;
        try {
            entryReader = Utils.ioResult(reader.readNext(1)).get(0);
        } catch (EndOfLogSegmentException eol) {
            done = true;
            continue;
        }
        LogRecordWithDLSN record = entryReader.nextRecord();
        while (null != record) {
            if (!record.isControl()) {
                DLMTestUtil.verifyLogRecord(record);
                assertEquals(txId, record.getTransactionId());
                ++txId;
            }
            DLSN dlsn = record.getDlsn();
            assertEquals(1L, dlsn.getLogSegmentSequenceNo());
            assertEquals(entryId, dlsn.getEntryId());
            record = entryReader.nextRecord();
        }
        ++entryId;
    }
    assertEquals(21, txId);
    assertFalse(reader.hasCaughtUpOnInprogress());
    Utils.close(reader);
}
 

@Test(timeout = 60000)
public void testMaxPrefetchEntriesSmallBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(2);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 10) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals(10, reader.readAheadEntries.size());
    assertEquals(10, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 10) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    assertEquals(10, reader.readAheadEntries.size());
    assertEquals(11, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}
 

@Test(timeout = 60000)
public void testMaxPrefetchEntriesLargeBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(5);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(5, reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(6, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}
 

@Test(timeout = 60000)
public void testMaxPrefetchEntriesSmallSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(20);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 5);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < (reader.getLastAddConfirmed() + 1)) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals((reader.getLastAddConfirmed() + 1), reader.readAheadEntries.size());
    assertEquals((reader.getLastAddConfirmed() + 1), reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    assertEquals(reader.getLastAddConfirmed(), reader.readAheadEntries.size());
    assertEquals((reader.getLastAddConfirmed() + 1), reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}