net.openhft.chronicle.wire.Wire Java Examples

public static NavigableMap<Integer, Wire> deserializeEvents(final OrderCommand cmd) {

        final Map<Integer, List<MatcherTradeEvent>> sections = new HashMap<>();
        cmd.processMatcherEvents(evt -> sections.computeIfAbsent(evt.section, k -> new ArrayList<>()).add(evt));

        NavigableMap<Integer, Wire> result = new TreeMap<>();

        sections.forEach((section, events) -> {
            final long[] dataArray = events.stream()
                    .flatMap(evt -> Stream.of(
                            evt.matchedOrderId,
                            evt.matchedOrderUid,
                            evt.price,
                            evt.size,
                            evt.bidderHoldPrice))
                    .mapToLong(s -> s)
                    .toArray();

            final Wire wire = SerializationUtils.longsToWire(dataArray);

            result.put(section, wire);
        });


        return result;
    }
 

private void removeEOF(Path path) throws IOException {
    long blockSize = 64 << 10;
    long chunkSize = OS.pageAlign(blockSize);
    long overlapSize = OS.pageAlign(blockSize / 4);
    final MappedBytes mappedBytes = MappedBytes.mappedBytes(path.toFile(), chunkSize, overlapSize, false);
    mappedBytes.reserve(test);
    try {
        final Wire wire = WireType.BINARY_LIGHT.apply(mappedBytes);
        final Bytes<?> bytes = wire.bytes();
        bytes.readLimitToCapacity();
        bytes.readSkip(4);
        // move past header
        try (final SingleChronicleQueueStore qs = loadStore(wire)) {
            assertNotNull(qs);
            long l = qs.writePosition();
            long len = Wires.lengthOf(bytes.readVolatileInt(l));
            long eofOffset = l + len + 4L;
            bytes.writePosition(eofOffset);
            bytes.writeInt(0);
        }
    } finally {
        mappedBytes.release(test);
    }
}
 

@Override
public <T> T loadData(long snapshotId,
                      SerializedModuleType type,
                      int instanceId,
                      Function<BytesIn, T> initFunc) {

    final Path path = resolveSnapshotPath(snapshotId, type, instanceId);

    log.debug("Loading state from {}", path);
    try (final InputStream is = Files.newInputStream(path, StandardOpenOption.READ);
         final InputStream bis = new BufferedInputStream(is);
         final LZ4FrameInputStream lz4is = new LZ4FrameInputStream(bis)) {

        // TODO improve reading algorithm
        final InputStreamToWire inputStreamToWire = new InputStreamToWire(WireType.RAW, lz4is);
        final Wire wire = inputStreamToWire.readOne();

        log.debug("start de-serializing...");

        AtomicReference<T> ref = new AtomicReference<>();
        wire.readBytes(bytes -> ref.set(initFunc.apply(bytes)));

        return ref.get();

    } catch (final IOException ex) {
        log.error("Can not read snapshot file: ", ex);
        throw new IllegalStateException(ex);
    }
}
 

static void addToEndOfCache(Wire wire2) {
    Bytes<?> bytes = wire2.bytes();
    long addr = bytes.addressForWrite(bytes.writePosition());
    int pad = (int) (64 - (addr & 63));
    if (pad < 64)
        wire2.addPadding(pad);
}
 

@NotNull
private Wire toWire(int key) {
    final MyData myData = new MyData();
    myData.key = key;
    myData.value = Integer.toString(key);
    Wire result = WireType.BINARY.apply(Bytes.elasticByteBuffer());

    try (final DocumentContext dc = result.writingDocument()) {
        dc.wire().getValueOut().typedMarshallable(myData);
    }

    return result;
}
 

/**
 * the following steps
 * <p>
 * (1) write to a queue
 * (2) force and end for file marker
 * (3) write to the queue again, this will cause it to be written to tomorrows .cq4 file
 * (4) delete todays queue files, that was created in step (1)
 * (5) create a new instance of chronicle queue ( same directory ) as step 1
 * (6) create a tailer toEnd
 * (6) write to this queue created in (5)
 * (7) when you now try to read from this queue you will not be able to read back what you have just written in (6)
 */
@Test
public void test() {
    File tmpDir = getTmpDir();
    File[] files;
    try (ChronicleQueue writeQ = ChronicleQueue.singleBuilder(tmpDir).build()) {
        ExcerptAppender appender = writeQ.acquireAppender();
        long wp;
        Wire wire;

        try (DocumentContext dc = appender.writingDocument()) {
            wire = dc.wire();
            wire.write().text("hello world");
            Bytes<?> bytes = wire.bytes();
            wp = bytes.writePosition();
        }

        File dir = new File(appender.queue().fileAbsolutePath());
        files = dir.listFiles(pathname -> pathname.getAbsolutePath().endsWith(".cq4"));

        wire.bytes().writeInt(wp, Wires.END_OF_DATA);
        appender.writeText("hello world  2");
    }

    Assert.assertEquals(files.length, 1);
    File file = files[0];
    file.delete();

    try (ChronicleQueue q = ChronicleQueue.singleBuilder(tmpDir).build()) {
        ExcerptTailer excerptTailer = q.createTailer().toEnd();
        q.acquireAppender()
                .writeText(EXPECTED);
        Assert.assertEquals(EXPECTED, excerptTailer.readText());
    }
}
 

@Test
@Ignore("https://github.com/OpenHFT/Chronicle-Wire/issues/165")
public void testWriteMarshallableBinary() {
    final SingleChronicleQueueBuilder builder = SingleChronicleQueueBuilder.single("test").rollCycle(HOURLY);

    builder.build().close();
    final Wire wire = Wires.acquireBinaryWire();
    wire.write().typedMarshallable(builder);

    System.err.println(wire.bytes().toHexString());

    SingleChronicleQueueBuilder builder2 = wire.read().typedMarshallable();
    builder2.build().close();
}
 

@Test
public void testNative() {
    if (!OS.isMacOSX())
        return;

    Bytes<ByteBuffer> bytes = Bytes.elasticByteBuffer();
    try {

        BatchAppenderNative batchAppenderNative = new BatchAppenderNative();

        // this will append a message in wire of hello world
        long result = batchAppenderNative.writeMessages(bytes.addressForWrite(0), bytes.realCapacity(), 1);

        int len = (int) result;
        int count = (int) (result >> 32);
        bytes.readLimit(len);

        Assert.assertEquals(16, len);
        Assert.assertEquals(1, count);

        Wire w = WireType.BINARY.apply(bytes);

        for (int i = 0; i < count; i++) {
            try (DocumentContext dc = w.readingDocument()) {
                Assert.assertEquals("hello world", dc.wire().getValueIn().text());
            }
        }
    } finally {
        bytes.releaseLast();
    }
}
 

@Nullable
static SingleChronicleQueueStore loadStore(@NotNull Wire wire) {
    final StringBuilder eventName = new StringBuilder();
    wire.readEventName(eventName);
    if (eventName.toString().equals(MetaDataKeys.header.name())) {
        final SingleChronicleQueueStore store = wire.read().typedMarshallable();
        if (store == null) {
            throw new IllegalArgumentException("Unable to load wire store");
        }
        return store;
    }

    Jvm.warn().on(RollEOFTest.class, "Unable to load store file from input. Queue file may be corrupted.");
    return null;
}
 

@NotNull
private TableStore<T> writeTableStore(MappedBytes bytes, Wire wire) {
    TableStore<T> store = new SingleTableStore<>(wireType, bytes, metadata);
    wire.writeEventName("header").object(store);
    wire.updateFirstHeader();
    return store;
}
 

@NotNull
private TableStore<T> readTableStore(Wire wire) throws StreamCorruptedException {
    wire.readFirstHeader();

    StringBuilder name = Wires.acquireStringBuilder();
    ValueIn valueIn = wire.readEventName(name);
    if (StringUtils.isEqual(name, MetaDataKeys.header.name())) {
        @NotNull TableStore<T> existing = Objects.requireNonNull(valueIn.typedMarshallable());
        metadata.overrideFrom(existing.metadata());
        return existing;
    } else {
        throw new StreamCorruptedException("The first message should be the header, was " + name);
    }
}
 

private static void writeMessage(Wire wire, int messageSize) {
    Bytes<?> bytes = wire.bytes();
    long wp = bytes.writePosition();
    long addr = bytes.addressForWrite(wp);
    Memory memory = OS.memory();
    for (int i = 0; i < messageSize; i += 16) {
        memory.writeLong(addr + i, 0L);
        memory.writeLong(addr + i + 8, 0L);
    }

    bytes.writeSkip(messageSize);
    bytes.writeLong(wp, System.nanoTime());
}
 

private static void runInner(Histogram transportTime, Histogram readTime, ExcerptTailer tailer) {
    Jvm.safepoint();
    if (tailer.peekDocument()) {
        if (counter++ < 1000) {
            Jvm.safepoint();
            return;
        }
    }
    if (counter > 0)
        Jvm.safepoint();
    else
        Jvm.safepoint();
    counter = 0;
    try (DocumentContext dc = tailer.readingDocument(false)) {
        Jvm.safepoint();
        if (!dc.isPresent()) {
            return;
        }
        long transport = System.nanoTime();
        Jvm.safepoint();
        Wire wire = dc.wire();
        Bytes<?> bytes = wire.bytes();
        long start = readMessage(bytes);
        long end = System.nanoTime();
        transportTime.sample(transport - start);
        readTime.sample(end - transport);
    }
    Jvm.safepoint();
}
 

@Override
    public void onEightyByteMessage(final EightyByteMessage message) {

        final ExcerptAppender appender = outputSupplier.get();
//        DebugTimestamps.operationStart(DebugTimestamps.Operation.GET_WRITING_DOCUMENT);
        @NotNull DocumentContext context = appender.writingDocument();
//        DebugTimestamps.operationEnd(DebugTimestamps.Operation.GET_WRITING_DOCUMENT);
        try {
            Wire wire = context.wire();
            // log write
//            DebugTimestamps.operationStart(DebugTimestamps.Operation.WRITE_EVENT);
            try {
                wire.write(MethodReader.HISTORY).marshallable(MessageHistory.get());
                final ValueOut valueOut = wire.writeEventName("onEightyByteMessage");
                valueOut.object(EightyByteMessage.class, message);
                wire.padToCacheAlign();
            } finally {
//                DebugTimestamps.operationEnd(DebugTimestamps.Operation.WRITE_EVENT);
            }
        } finally {

//            DebugTimestamps.operationStart(DebugTimestamps.Operation.CLOSE_CONTEXT);
            try {
                context.close();
            } finally {
//                DebugTimestamps.operationEnd(DebugTimestamps.Operation.CLOSE_CONTEXT);
            }
        }
    }
 

/**
 * Creates a new Excerpt containing and index which will be 1L << 17L bytes long, This method is used for creating
 * both the primary and secondary indexes. Chronicle Queue uses a root primary index ( each entry in the primary
 * index points to a unique a secondary index. The secondary index only records the addressForRead of every 64th except,
 * the except are linearly scanned from there on.
 *
 * @return the addressForRead of the Excerpt containing the usable index, just after the header
 */
long newIndex() {
    final ByteableLongArrayValues array = longArray.get();

    final long size = array.sizeInBytes(NUMBER_OF_ENTRIES_IN_EACH_INDEX);
    final Bytes buffer = NativeBytes.nativeBytes(size);
    buffer.zeroOut(0, size);

    final Wire wire = WireUtil.createWire(this.builder.wireType(), buffer);
    wire.write(() -> "index").int64array(NUMBER_OF_ENTRIES_IN_EACH_INDEX);
    buffer.flip();

    return appendMetaDataReturnAddress(buffer);
}
 

public StatelessTailer(ChronicleQueue chronicleQueue,
                       Function<Bytes, Wire> wireFunction,
                       StatelessRawBytesTailer statelessRawBytesTailer) {
    this.wireFunction = wireFunction;
    this.statelessRawBytesTailer = statelessRawBytesTailer;
    this.chronicleQueue = chronicleQueue;
}
 

public StatelessChronicleQueue(Function<Bytes, Wire> wireFunction,
                               StatelessRawBytesTailer statelessRawBytesTailer,
                               StatelessRawBytesAppender statelessRawBytesAppender) {
    this.wireFunction = wireFunction;
    this.statelessRawBytesTailer = statelessRawBytesTailer;
    this.statelessRawBytesAppender = statelessRawBytesAppender;
}
 

public <Q extends ReportQuery<R>, R extends ReportResult> CompletableFuture<R> processReport(final Q query, final int transferId) {
    return submitQueryAsync(
            query,
            transferId,
            cmd -> query.createResult(
                    OrderBookEventsHelper.deserializeEvents(cmd).values().parallelStream().map(Wire::bytes)));
}
 

private void readDocument(Wire inMetaDataWire) throws IOException {
    ByteBuffer inBuff = (ByteBuffer) inMetaDataWire.bytes().underlyingObject();

    // write the data to the socket
    long start = inMetaDataWire.bytes().writePosition();

    while (inBuff.position() < 4 + start)
        client.read(inBuff);

    int len = Wires.lengthOf(inMetaDataWire.bytes().readInt(start));
    inMetaDataWire.bytes().readLimit(start + 4 + len);
    while (inBuff.position() < 4 + len + start)
        client.read(inBuff);
}
 

@Override
protected Wire initialiseOutWire(Bytes<?> out, @NotNull WireType wireType) {
    Wire wire = super.initialiseOutWire(out, wireType);
    output = wire.methodWriter(outClass);
    return wire;

}
 

public ClientWiredExcerptAppenderStateless(ClientWiredChronicleQueueStateless queue,
                                           ClientWiredStatelessTcpConnectionHub hub,
                                           Function<Bytes, Wire> wireWrapper) {
    super(queue.name(), hub, "QUEUE", 0);
    this.queue = queue;
    this.csp = "//" + queue.name() + "?view=QUEUE";
    QueueAppenderResponse qar = (QueueAppenderResponse) proxyReturnMarshallable(EventId.createAppender);
    this.cid = qar.getCid();
    this.wire = wireWrapper.apply(source);
}
 

private void readDocument(@NotNull Wire inMetaDataWire) throws IOException {
    @Nullable ByteBuffer inBuff = (ByteBuffer) inMetaDataWire.bytes().underlyingObject();

    // write the data to the socket
    long start = inMetaDataWire.bytes().readPosition();
    while (inBuff.position() + start < 4)
        client.read(inBuff);

    inMetaDataWire.bytes().writePosition(inBuff.position());
    int len = Wires.lengthOf(inMetaDataWire.bytes().readInt(start));
    while (inBuff.position() < 4 + len + start)
        client.read(inBuff);
}
 

/**
 * @return ByteBuffer, with self bootstrapping header in [position, limit) range
 */
private static <K, V> ByteBuffer writeHeader(
        FileChannel fileChannel, VanillaChronicleMap<K, V, ?> map) throws IOException {
    ByteBuffer headerBuffer = ByteBuffer.allocate(
            SELF_BOOTSTRAPPING_HEADER_OFFSET + MAX_BOOTSTRAPPING_HEADER_SIZE);
    headerBuffer.order(LITTLE_ENDIAN);

    Bytes<ByteBuffer> headerBytes = Bytes.wrapForWrite(headerBuffer);
    headerBytes.writePosition(SELF_BOOTSTRAPPING_HEADER_OFFSET);
    Wire wire = new TextWire(headerBytes);
    wire.getValueOut().typedMarshallable(map);

    int headerLimit = (int) headerBytes.writePosition();
    int headerSize = headerLimit - SELF_BOOTSTRAPPING_HEADER_OFFSET;
    // First set readiness bit to READY, to compute checksum correctly
    //noinspection PointlessBitwiseExpression
    headerBuffer.putInt(SIZE_WORD_OFFSET, READY | DATA | headerSize);

    long checksum = headerChecksum(headerBuffer, headerSize);
    headerBuffer.putLong(HEADER_OFFSET, checksum);

    // Set readiness bit to NOT_COMPLETE, because the Chronicle Map instance is not actually
    // ready yet
    //noinspection PointlessBitwiseExpression
    headerBuffer.putInt(SIZE_WORD_OFFSET, NOT_COMPLETE | DATA | headerSize);

    // Write the size-prefixed blob to the file
    headerBuffer.position(0).limit(headerLimit);
    writeFully(fileChannel, 0, headerBuffer);

    headerBuffer.position(SELF_BOOTSTRAPPING_HEADER_OFFSET);
    return headerBuffer;
}
 

public StatelessExcerpt(ChronicleQueue chronicleQueue, Function<Bytes, Wire> wireFunction,
                        StatelessRawBytesTailer statelessRawBytesTailer) {
    this.wireFunction = wireFunction;
    this.statelessRawBytesTailer = statelessRawBytesTailer;
    this.chronicleQueue = chronicleQueue;
}
 

public Wire getWire() {
    wire.clear();
    return wire;
}
 

public static Wire longsToWire(long[] dataArray) {

        final int sizeInBytes = dataArray.length * 8;
        final ByteBuffer byteBuffer = ByteBuffer.allocate(sizeInBytes);
        byteBuffer.asLongBuffer().put(dataArray);

        final byte[] bytesArray = new byte[sizeInBytes];
        byteBuffer.get(bytesArray);

        //log.debug(" section {} -> {}", section, bytes);

        final Bytes<ByteBuffer> bytes = Bytes.elasticHeapByteBuffer(sizeInBytes);
        bytes.ensureCapacity(sizeInBytes);

        bytes.write(bytesArray);

        return WireType.RAW.apply(bytes);
    }
 

public static Wire longsLz4ToWire(long[] dataArray, int longsTransfered) {

//        log.debug("long dataArray.len={} longsTransfered={}", dataArray.length, longsTransfered);

        final ByteBuffer byteBuffer = ByteBuffer.allocate(longsTransfered * 8);
        byteBuffer.asLongBuffer().put(dataArray, 0, longsTransfered);

        final int originalSizeBytes = byteBuffer.getInt();

        final ByteBuffer uncompressedByteBuffer = ByteBuffer.allocate(originalSizeBytes);

        final LZ4FastDecompressor lz4FastDecompressor = LZ4Factory.fastestInstance().fastDecompressor();

        lz4FastDecompressor.decompress(byteBuffer, byteBuffer.position(), uncompressedByteBuffer, uncompressedByteBuffer.position(), originalSizeBytes);

        final Bytes<ByteBuffer> bytes = Bytes.wrapForRead(uncompressedByteBuffer);

        return WireType.RAW.apply(bytes);
    }
 

public WireEchoRequestHandler(@NotNull Function<Bytes, Wire> bytesToWire) {
    super(bytesToWire);
}
 

@Override
protected Wire initialiseInWire(@NotNull WireType wireType, Bytes<?> in) {
    Wire wire = super.initialiseInWire(wireType, in);
    reader = wire.methodReader(implSupplier.get());
    return wire;
}
 

@Nullable
Wire wireForIndex();