net.openhft.chronicle.wire.DocumentContext Java Examples

public static void main(String[] args) throws IOException {
    AvroHelper avro = new AvroHelper();

    String path = "queue";
    SingleChronicleQueue queue = SingleChronicleQueueBuilder.binary(path).build();
    ExcerptTailer tailer = queue.createTailer();

    while (true) {
        try (DocumentContext dc = tailer.readingDocument()) {
            if (dc.wire() == null)
                break;

            GenericRecord user = avro.readFromIS(dc.wire().bytes().inputStream());
            System.out.println("Read: " + user);
        }

    }

    System.out.println("All done");
}
 

@Test
   public void testDtoBytesMarshallable() {

       File tmp = DirectoryUtils.tempDir("abstractBytesMarshalTest");

       DtoBytesMarshallable dto = new DtoBytesMarshallable();

       dto.age = 45;
       dto.name.append("rob");

       try (ChronicleQueue q = ChronicleQueue.singleBuilder(tmp).build()) {

           try (DocumentContext dc = q.acquireAppender().writingDocument()) {
               dc.wire().write("who").object(dto);
           }

           try (DocumentContext dc = q.createTailer().readingDocument()) {

               DtoBytesMarshallable who = (DtoBytesMarshallable) dc.wire().read("who").object();
               Assert.assertEquals("!net.openhft.chronicle.queue.DtoBytesMarshallableTest$DtoBytesMarshallable {\n" +
                       "  name: rob,\n" +
                       "  age: 45\n" +
                       "}\n", who.toString());
           }
       }
}
 

@Test
public void uninitialisedTailerCreatedBeforeFirstAppendWithDirectionNoneShouldNotFindDocument() {
    final AtomicLong clock = new AtomicLong(System.currentTimeMillis());
    String path = OS.TARGET + "/" + getClass().getSimpleName() + "-" + System.nanoTime();
    try (final ChronicleQueue queue = SingleChronicleQueueBuilder.single(path).timeProvider(clock::get).testBlockSize()
            .rollCycle(RollCycles.TEST_SECONDLY).build()) {

        final ExcerptTailer tailer = queue.createTailer();
        tailer.direction(TailerDirection.NONE);

        final ExcerptAppender excerptAppender = queue.acquireAppender();
        for (int i = 0; i < 10; i++) {
            excerptAppender.writeDocument(i, (out, value) -> {
                out.int32(value);
            });
        }

        DocumentContext document = tailer.readingDocument();
        assertFalse(document.isPresent());
    }
}
 

@Override
public synchronized Integer call() {

    try (final DocumentContext dc = tailer.readingDocument()) {

        System.out.println("index=" + Long.toHexString(dc.index()));
        if (!dc.isPresent())
            return documentsRead;

        StringBuilder sb = Wires.acquireStringBuilder();
        dc.wire().bytes().parse8bit(sb, StopCharTesters.ALL);

        String readText = sb.toString();
        if (java.util.Objects.equals(readText, "")) {
            return documentsRead;
        }
        System.out.println("Read a document " + readText);
        documentsRead++;

    }
    return documentsRead;
}
 

private void appendToQueue() {
    ExcerptAppender appender = queue0.acquireAppender();
    try {

        for (int i = 0; i < BATCH_SIZE; i++) {
            try (DocumentContext dc = appender.writingDocument()) {
                long seq = appender.queue().rollCycle().toSequenceNumber(dc.index());
                //   System.out.println("write=" + Long.toHexString(dc.index()));
                dc.wire().write("value").writeLong(seq);
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(-1);
    }
}
 

private static long[] writeMessagesStoreIndices(final ExcerptAppender appender, final ExcerptTailer tailer) {
    final long[] indices = new long[RECORD_COUNT];
    for (int i = 0; i < RECORD_COUNT; i++) {
        try (final DocumentContext ctx = appender.writingDocument()) {
            ctx.wire().getValueOut().int32(i);
        }
    }

    for (int i = 0; i < RECORD_COUNT; i++) {
        try (final DocumentContext ctx = tailer.readingDocument()) {
            assertTrue("Expected record at index " + i, ctx.isPresent());
            indices[i] = tailer.index();
        }
    }
    return indices;
}
 

@Override
public void run(long startTimeNS) {
    datum.setValue10(startTimeNS);

    try (DocumentContext dc = appender.writingDocument()) {
        dc.wire().bytes().write(datumBytes);
        //datum.writeMarshallable(dc.wire().bytes());
    }

    try (DocumentContext dc = tailer.readingDocument()) {
        if (dc.wire() != null) {
            datumWrite.writePosition(0);
            dc.wire().readBytes(datumWrite);
            //datum.readMarshallable(dc.wire().bytes());
            jlbh.sample(System.nanoTime() - datum.getValue10());
        }
    }
}
 

@Override
public void init(JLBH jlbh) {
    IOTools.deleteDirWithFiles("replica", 10);

    sourceQueue = single("replica").build();
    sinkQueue = single("replica").build();
    appender = sourceQueue.acquireAppender();
    tailer = sinkQueue.createTailer();
    new Thread(() -> {
        Datum datum2 = new Datum();
        while (true) {
            try (DocumentContext dc = tailer.readingDocument()) {
                if (dc.wire() == null)
                    continue;
                datum2.readMarshallable(dc.wire().bytes());
                jlbh.sample(System.nanoTime() - datum2.ts);
            }
        }
    }).start();
}
 

private long doRead(@NotNull ExcerptTailer tailer, int expected) {
    int[] i = {0};
    long t_index = 0;
    while (true) {
        try (DocumentContext dc = tailer.readingDocument()) {
            if (!dc.isPresent())
                break;
            t_index = tailer.index();
            dc.wire().read("log").marshallable(m -> {
                String msg = m.read("msg").text();
                assertNotNull(msg);
                System.out.println("msg:" + msg);
                i[0]++;
            });
        }
    }
    assertEquals(expected, i[0]);
    return t_index;
}
 

private void createQueueAndWriteData(TimeProvider timeProvider, File path) {

        try (final ChronicleQueue queue = SingleChronicleQueueBuilder
                .binary(path)
                .testBlockSize()
                .rollCycle(RollCycles.TEST_DAILY)
                .timeProvider(timeProvider)
                .build()) {

            ExcerptAppender excerptAppender = queue.acquireAppender();

            try (DocumentContext dc = excerptAppender.writingDocument(false)) {
                dc.wire().write(() -> "test").int64(0);
            }
        }
    }
 

private static long doReadBad(@NotNull ExcerptTailer tailer, int expected, boolean additionalClose) {
    int[] i = {0};
    long t_index = 0;
    while (true) {
        try (DocumentContext dc = tailer.readingDocument()) {
            if (!dc.isPresent())
                break;
            t_index = tailer.index();

            dc.wire().read("log").marshallable(m -> {
                String msg = m.read("msg").text();
                assertNotNull(msg);
                i[0]++;
            });
            if (additionalClose) {
                dc.close();
            }
        }
    }
    assertEquals(expected, i[0]);
    return t_index;
}
 

@Test
public void testReadFromReadOnlyChronicle() {
    try (ChronicleQueue out = SingleChronicleQueueBuilder
            .binary(chroniclePath)
            .testBlockSize()
            .readOnly(!OS.isWindows())
            .build()) {
        // check dump
        assertTrue(out.dump().length() > 1);
        // and tailer
        ExcerptTailer tailer = out.createTailer();
        assertEquals(STR1, tailer.readText());
        try (DocumentContext dc = tailer.readingDocument()) {
            assertEquals(STR2, dc.wire().bytes().readUtf8());
            // even though this is read-only we can still call dc.wire().bytes().write... which causes java.lang.InternalError
            // Fixing this in a type-safe manner would require on Read/WriteDocumentContext to return WireIn/WireOut
        }
    }
}
 

@Test
    public void test() {

        ValueStringArray value = new ValueStringArray();
        value.setCsArrItem(1, EXPECTED);

        // this is the directory the queue is written to
        final File dataDir = DirectoryUtils.tempDir(DumpQueueMainTest.class.getSimpleName());

        try (final ChronicleQueue queue = SingleChronicleQueueBuilder.binary(dataDir).build()) {

            try (DocumentContext dc = queue.acquireAppender().writingDocument()) {
                dc.wire().write("data").marshallable(value);
            }

            try (DocumentContext dc = queue.createTailer().readingDocument()) {
                dc.wire().read("data").marshallable(using);
                CharSequence actual = using.getCsArr().getCharSequenceWrapperAt(1).getCharSequence();
//                System.out.println(actual);
                Assert.assertEquals(EXPECTED, actual.toString());
            }
 }

    }
 

private static Data read(final ExcerptTailer tailer) throws IOException {
    try (final DocumentContext dc = tailer.readingDocument()) {
        if (!dc.isPresent()) {
            return null;
        }

        final ObjectInput in = dc.wire().objectInput();
        return new Data(in.readInt());
    }
}
 

private void appendToQueue(final ChronicleQueue queue) {
    for (int i = 0; i < 31; i++) {
        final ExcerptAppender appender = queue.acquireAppender();
        if (queue.isClosed())
            return;
        try (final DocumentContext dc = appender.writingDocument()) {
            dc.wire().write("foo");
        }
    }
}
 

@Test
public void testEntryCount() {
    try (SingleChronicleQueue queue = ChronicleQueue.singleBuilder(dataDirectory).build()) {
        assertEquals(0, queue.entryCount());

        try (DocumentContext dc = queue.acquireAppender().writingDocument()) {
            dc.wire().write("test").text("value");
        }

        assertEquals(1, queue.entryCount());
    }
}
 

@Test
public void shouldHandleCycleRollWhenInReadOnlyMode() {
    final MutableTimeProvider timeProvider = new MutableTimeProvider();
    try (ChronicleQueue queue = build(createQueue(dataDirectory, RollCycles.MINUTELY, 0, "cycleRoll", false).
            timeProvider(timeProvider))) {

        final StringEvents events = queue.acquireAppender().methodWriterBuilder(StringEvents.class).build();
        timeProvider.setTime(System.currentTimeMillis());
        events.onEvent("firstEvent");
        timeProvider.addTime(2, TimeUnit.MINUTES);
        events.onEvent("secondEvent");

        try (final ChronicleQueue readerQueue = build(createQueue(dataDirectory, RollCycles.MINUTELY, 0, "cycleRoll", true).
                timeProvider(timeProvider))) {

            final ExcerptTailer tailer = readerQueue.createTailer();
            tailer.toStart();
            try (final DocumentContext context = tailer.readingDocument()) {
                assertTrue(context.isPresent());
            }
            tailer.toEnd();
            try (final DocumentContext context = tailer.readingDocument()) {
                assertFalse(context.isPresent());
            }
        }
    }
}
 

@Test
public void shouldAppendToExistingQueueFile() throws IOException {
    try (final SingleChronicleQueue queue = createQueue()) {
        final ExcerptAppender appender = queue.acquireAppender();

        appender.writeDocument(1, (w, i) -> {
            w.int32(i);
        });

        final ExcerptTailer tailer = queue.createTailer();
        try (final DocumentContext context = tailer.readingDocument()) {
            assertTrue(context.isPresent());
        }

        assertQueueFileCount(queue.path.toPath(), 1);

        assertFalse(tailer.readingDocument().isPresent());

        appender.writeDocument(2, (w, i) -> {
            w.int32(i);
        });

        assertQueueFileCount(queue.path.toPath(), 1);
        try (final DocumentContext context = tailer.readingDocument()) {
            assertTrue(context.isPresent());
        }
    }
}
 

/**
 * the following steps
 * <p>
 * (1) write to a queue
 * (2) force and end for file marker
 * (3) write to the queue again, this will cause it to be written to tomorrows .cq4 file
 * (4) delete todays queue files, that was created in step (1)
 * (5) create a new instance of chronicle queue ( same directory ) as step 1
 * (6) create a tailer toEnd
 * (6) write to this queue created in (5)
 * (7) when you now try to read from this queue you will not be able to read back what you have just written in (6)
 */
@Test
public void test() {
    File tmpDir = getTmpDir();
    File[] files;
    try (ChronicleQueue writeQ = ChronicleQueue.singleBuilder(tmpDir).build()) {
        ExcerptAppender appender = writeQ.acquireAppender();
        long wp;
        Wire wire;

        try (DocumentContext dc = appender.writingDocument()) {
            wire = dc.wire();
            wire.write().text("hello world");
            Bytes<?> bytes = wire.bytes();
            wp = bytes.writePosition();
        }

        File dir = new File(appender.queue().fileAbsolutePath());
        files = dir.listFiles(pathname -> pathname.getAbsolutePath().endsWith(".cq4"));

        wire.bytes().writeInt(wp, Wires.END_OF_DATA);
        appender.writeText("hello world  2");
    }

    Assert.assertEquals(files.length, 1);
    File file = files[0];
    file.delete();

    try (ChronicleQueue q = ChronicleQueue.singleBuilder(tmpDir).build()) {
        ExcerptTailer excerptTailer = q.createTailer().toEnd();
        q.acquireAppender()
                .writeText(EXPECTED);
        Assert.assertEquals(EXPECTED, excerptTailer.readText());
    }
}
 

private int write(ExcerptAppender appender) {
    int value;
    try (DocumentContext writingDocument = appender.writingDocument()) {
        final long documentAcquireTimestamp = System.nanoTime();
        value = wrote.getAndIncrement();
        ValueOut valueOut = writingDocument.wire().getValueOut();
        // make the message longer
        valueOut.int64(documentAcquireTimestamp);
        for (int i = 0; i < NUMBER_OF_INTS; i++) {
            valueOut.int32(value);
        }
        writingDocument.wire().padToCacheAlign();
    }
    return value;
}
 

private static void expectValue(final int expectedValue, final ExcerptTailer tailer) {

        try (final DocumentContext documentContext = tailer.readingDocument()) {
            assertTrue(documentContext.isPresent());
            assertEquals(expectedValue, documentContext.wire().getValueIn().int32());
        }
 }
 

private Future<RecordInfo> attemptToWriteDocument(final ChronicleQueue queue) throws InterruptedException {
    final CountDownLatch startedLatch = new CountDownLatch(1);
    final Future<RecordInfo> future = executorService.submit(() -> {
        thread = Thread.currentThread();
        final int counterValue;
        startedLatch.countDown();
        try (final DocumentContext documentContext = queue.acquireAppender().writingDocument()) {
            counterValue = counter.getAndIncrement();
            documentContext.wire().getValueOut().int32(counterValue);
        }
        return new RecordInfo(counterValue);
    });
    assertTrue("Task did not start", startedLatch.await(1, TimeUnit.MINUTES));
    return future;
}
 

private static void runInner(Histogram transportTime, Histogram readTime, ExcerptTailer tailer) {
    Jvm.safepoint();
    if (tailer.peekDocument()) {
        if (counter++ < 1000) {
            Jvm.safepoint();
            return;
        }
    }
    if (counter > 0)
        Jvm.safepoint();
    else
        Jvm.safepoint();
    counter = 0;
    try (DocumentContext dc = tailer.readingDocument(false)) {
        Jvm.safepoint();
        if (!dc.isPresent()) {
            return;
        }
        long transport = System.nanoTime();
        Jvm.safepoint();
        Wire wire = dc.wire();
        Bytes<?> bytes = wire.bytes();
        long start = readMessage(bytes);
        long end = System.nanoTime();
        transportTime.sample(transport - start);
        readTime.sample(end - transport);
    }
    Jvm.safepoint();
}
 

@Test
public void tailerShouldBeAbleToMoveBackwardFromEndOfCycle() throws IOException {
    assumeFalse(OS.isWindows());
    try (final ChronicleQueue queue = createQueue(path, clock::get)) {
        final ExcerptAppender appender = queue.acquireAppender();
        // generate some cycle files
        range(0, 5).forEach(i -> {
            try (final DocumentContext ctx = appender.writingDocument()) {
                ctx.wire().write().int32(i);
                clock.addAndGet(TimeUnit.SECONDS.toMillis(10L));
            }
        });
    }

    // remove all but the first file
    final Path firstFile =
            Files.list(this.path.toPath())
                    .sorted(Comparator.comparing(Path::toString))
                    .findFirst()
                    .orElseThrow(AssertionError::new);
    Files.list(this.path.toPath())
            .filter(p -> !p.equals(firstFile))
            .forEach(TailerIndexingQueueTest::deleteFile);

    try (final ChronicleQueue queue = createQueue(path, SystemTimeProvider.INSTANCE)) {
        final ExcerptTailer tailer = queue.createTailer().toEnd();
        // move to END_OF_CYCLE
        try (final DocumentContext readCtx = tailer.readingDocument()) {
            assertFalse(readCtx.isPresent());
        }
        assertEquals(TailerState.END_OF_CYCLE, tailer.state());

        tailer.direction(TailerDirection.BACKWARD);

        tailer.toEnd();
        assertTrue(tailer.readingDocument().isPresent());
    }
}
 

@NotNull
private List<Integer> fillResults(@NotNull ExcerptTailer tailer, @NotNull List<Integer> results) {
    for (int i = 0; i < 10; i++) {

        try (DocumentContext documentContext = tailer.readingDocument()) {
            if (!documentContext.isPresent())
                break;
            results.add(documentContext.wire().read(() -> "msg").int32());
        }
    }
    return results;
}
 

@NotNull
private Wire toWire(int key) {
    final MyData myData = new MyData();
    myData.key = key;
    myData.value = Integer.toString(key);
    Wire result = WireType.BINARY.apply(Bytes.elasticByteBuffer());

    try (final DocumentContext dc = result.writingDocument()) {
        dc.wire().getValueOut().typedMarshallable(myData);
    }

    return result;
}
 

@Override
    public void onEightyByteMessage(final EightyByteMessage message) {

        final ExcerptAppender appender = outputSupplier.get();
//        DebugTimestamps.operationStart(DebugTimestamps.Operation.GET_WRITING_DOCUMENT);
        @NotNull DocumentContext context = appender.writingDocument();
//        DebugTimestamps.operationEnd(DebugTimestamps.Operation.GET_WRITING_DOCUMENT);
        try {
            Wire wire = context.wire();
            // log write
//            DebugTimestamps.operationStart(DebugTimestamps.Operation.WRITE_EVENT);
            try {
                wire.write(MethodReader.HISTORY).marshallable(MessageHistory.get());
                final ValueOut valueOut = wire.writeEventName("onEightyByteMessage");
                valueOut.object(EightyByteMessage.class, message);
                wire.padToCacheAlign();
            } finally {
//                DebugTimestamps.operationEnd(DebugTimestamps.Operation.WRITE_EVENT);
            }
        } finally {

//            DebugTimestamps.operationStart(DebugTimestamps.Operation.CLOSE_CONTEXT);
            try {
                context.close();
            } finally {
//                DebugTimestamps.operationEnd(DebugTimestamps.Operation.CLOSE_CONTEXT);
            }
        }
    }
 

@Test
public void testRead1() throws ExecutionException, InterruptedException {
    File path = DirectoryUtils.tempDir(getClass().getSimpleName());
    TestTimeProvider timeProvider = new TestTimeProvider();

    try (ChronicleQueue queue0 = SingleChronicleQueueBuilder
            .fieldlessBinary(path)
            .testBlockSize()
            .rollCycle(DAILY)
            .timeProvider(timeProvider).build()) {

        ParallelQueueObserver observer = new ParallelQueueObserver(queue0);

        final ExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
                new NamedThreadFactory("test"));

        try (ChronicleQueue queue = SingleChronicleQueueBuilder
                .fieldlessBinary(path)
                .testBlockSize()
                .rollCycle(DAILY)
                .timeProvider(timeProvider)
                .build()) {
            ExcerptAppender appender = queue.acquireAppender();

            Assert.assertEquals(0, (int) scheduledExecutorService.submit(observer::call).get());
            // two days pass
            timeProvider.add(TimeUnit.DAYS.toMillis(2));

            try (final DocumentContext dc = appender.writingDocument()) {
                dc.wire().write().text("Day 3 data");
            }
            Assert.assertEquals(1, (int) scheduledExecutorService.submit(observer::call).get());
            assertEquals(1, observer.documentsRead);

        }
        scheduledExecutorService.shutdown();
        scheduledExecutorService.awaitTermination(1, TimeUnit.SECONDS);
    }
}
 

/**
 * simply reads the csp,tid and payload and sends back the tid and payload
 *
 * @param in      the DocumentContext from the client
 * @param outWire the wire to be sent back to the server
 */
@Override
protected void onRead(@NotNull DocumentContext in,
                      @NotNull WireOut outWire) {

    if (in.isMetaData())
        outWire.writeDocument(true, meta -> meta.write(() -> "tid")
                .int64(in.wire().read(() -> "tid").int64()));
    else
        outWire.writeDocument(false, data -> data.write(() -> "payloadResponse")
                .text(in.wire().read(() -> "payload").text()));

}
 

/**
 * Add a test message with the given ExcerptAppender and return the index position of the entry
 *
 * @param appender ExceptAppender
 * @param msg      test message
 * @return index position of the entry
 */
private long appendEntry(@NotNull final ExcerptAppender appender, String msg) {
    DocumentContext dc = appender.writingDocument();
    try {
        dc.wire().write().text(msg);
        return dc.index();
    } finally {
        dc.close();
    }
}