net.openhft.chronicle.bytes.Bytes Java Examples

/**
 * Apply waiting messages and return false if there was none.
 *
 * @param bytes buffer to write to.
 */
@Override
public void applyAction(@NotNull Bytes bytes) {
    resetUsedByThread();

    if (this.bytes.readRemaining() > 0) {

        synchronized (lock()) {

            if (YamlLogging.showServerWrites())
                logBuffer();

            bytes.write(this.bytes);
            this.bytes.clear();
        }
    }
}
 

private static void readMessage(final ChronicleQueue queue,
                                final boolean manuallyReleaseResources,
                                final Consumer<ExcerptTailer> refHolder) {
    final Bytes<ByteBuffer> bytes = Bytes.elasticByteBuffer();
    try (final ExcerptTailer tailer = queue.createTailer()) {
        while (bytes.isEmpty()) {
            tailer.toStart().readBytes(bytes);
        }
        refHolder.accept(tailer);
        assertTrue(Math.signum(bytes.readInt()) >= 0);

        if (manuallyReleaseResources) {
            try {
                ((StoreTailer) tailer).releaseResources();
            } catch (RuntimeException e) {
                // ignore
            }
        }
    } finally {
        bytes.releaseLast();
    }
}
 

/**
 * @param bytes                the bytes to send
 * @param reattemptUponFailure if false - will only be sent if the connection is valid
 */
protected boolean sendBytes(@NotNull final Bytes bytes,
                            final boolean reattemptUponFailure) {

    if (reattemptUponFailure)
        hub.lock(hub::checkConnection);
    else if (!hub.isOpen())
        return false;

    if (!reattemptUponFailure) {
        hub.lock(() -> quietSendBytesAsyncWithoutLock(bytes));
        return true;
    }

    attempt(() -> {
        hub.lock(() -> quietSendBytesAsyncWithoutLock(bytes));
        return true;
    });

    return false;
}
 

public static void dump(File filename, PrintWriter pw) throws FileNotFoundException {
    MappedFile mappedFile = MappedFile.mappedFile(filename, 64 << 20, 16 << 20);
    Bytes bytes = mappedFile.bytes();
    pw.print("# Magic: ");
    for (int i = 0; i < 8; i++)
        pw.print((char) bytes.readUnsignedByte());
    pw.println();
    while (true) {
        long spb = bytes.readUnsignedInt();
        if (!Wires.isKnownLength(spb))
            break;
        pw.print("--- !");
        pw.print(SBP_TYPES[((int) (spb >>> 30))]);
        pw.println();
        long start = bytes.position();
        BytesUtil.toString(bytes, pw, start, start, start + Wires.lengthOf(spb));
        pw.println();
        bytes.skip(Wires.lengthOf(spb));
    }
    pw.flush();
}
 

private long write(long offset, @NotNull Bytes bytes) {
    long result = offset + bytes.remaining();
    offset %= capacity();

    long len = bytes.remaining();
    long endOffSet = nextOffset(offset, len);

    if (endOffSet >= offset) {
        this.buffer.write(offset, bytes);
        return result;
    }

    long limit = bytes.limit();

    bytes.limit(capacity() - offset);
    this.buffer.write(offset, bytes);

    bytes.position(bytes.limit());
    bytes.limit(limit);

    this.buffer.write(0, bytes);
    return result;
}
 

@NotNull
@Override
public ByteBuffer read(@NotNull Bytes in, long size, @Nullable ByteBuffer using) {
    if (size < 0L || size > (long) Integer.MAX_VALUE)
        throw new IllegalArgumentException("ByteBuffer size should be non-negative int, " +
                size + " given. Memory corruption?");
    int bufferCap = (int) size;
    if (using == null || using.capacity() < bufferCap) {
        using = ByteBuffer.allocate(bufferCap);
    } else {
        using.position(0);
        using.limit(bufferCap);
    }
    in.read(using);
    using.flip();
    return using;
}
 

@Test
public void sign2() {
    final String SIGN_PRIVATE = "B18E1D0045995EC3D010C387CCFEB984D783AF8FBB0F40FA7DB126D889F6DADD";
    Bytes privateKey = fromHex(SIGN_PRIVATE);
    Bytes publicKey = bytesWithZeros(32);
    Bytes secretKey = bytesWithZeros(64);
    Ed25519.privateToPublicAndSecret(publicKey, secretKey, privateKey);
    checkZeros(secretKey);
    checkZeros(privateKey);

    Bytes message = privateKey;
    Bytes signAndMsg = Bytes.allocateDirect(Ed25519.SIGNATURE_LENGTH + message.readRemaining());
    signAndMsg.writeSkip(Ed25519.SIGNATURE_LENGTH);
    signAndMsg.write(privateKey);
    Ed25519.sign(signAndMsg, secretKey);

    String SIGN_EXPECTED = "86b4707fadb1ef4613efadd12143cd9dffb2eac329c38923c03f9e315c3dd33bde1ef101137fbc403eb3f3d7ff283155053c667eb65908fe6fcd653eab550e0f";
    Bytes signExpected = fromHex(SIGN_EXPECTED + SIGN_PRIVATE);
    assertEquals(signExpected.toHexString(), signAndMsg.toHexString());

    signAndMsg.releaseLast();
}
 

@Test
public void timeParseSingleOrder() {
    String s = "8=FIX.4.2|9=130|35=D|34=659|49=BROKER04|56=REUTERS|52=20070123-19:09:43|38=1000|59=1|100=N|40=1|11=ORD10001|60=20070123-19:01:17|55=HPQ|54=1|21=2|10=004|";
    Bytes nb = Bytes.from(s.replace('|', '\u0001'));

    final AtomicInteger count = new AtomicInteger();
    FixSaxParser parser = new FixSaxParser(new MyFixHandler(count));
    int runs = 200000;
    for (int t = 0; t < 5; t++) {
        count.set(0);
        long start = System.nanoTime();
        for (int i = 0; i < runs; i++) {
            nb.readPosition(0);
            parser.parse(nb);
        }
        long time = System.nanoTime() - start;
        System.out.printf("Average parse time was %.2f us, fields per message %.2f%n",
                time / runs / 1e3, (double) count.get() / runs);
    }
}
 

@Override
public void accept(final WireIn wireIn, final Consumer<String> messageHandler) {
    final Bytes<?> serialisedMessage = wireIn.bytes();
    final byte dataFormatIndicator = serialisedMessage.readByte(serialisedMessage.readPosition());
    String text;

    if (isBinaryFormat(dataFormatIndicator)) {
        textConversionTarget.clear();
        final BinaryWire binaryWire = new BinaryWire(serialisedMessage);
        binaryWire.copyTo(wireType.apply(textConversionTarget));
        text = textConversionTarget.toString();
    } else {
        text = serialisedMessage.toString();
    }

    messageHandler.accept(text);
}
 

@NotNull
@Override
public A read(Bytes in, A using) {
    if (using == null)
        using = new A();
    using.str_ = in.readUtf8();
    int size = (int) in.readStopBit();
    if (size >= 0) {
        if (using.list_ == null) {
            using.list_ = new ArrayList<>(size);
        } else {
            using.list_.clear();
            if (using.list_ instanceof ArrayList)
                ((ArrayList) using.list_).ensureCapacity(size);
        }
        for (int i = 0; i < size; i++) {
            B b = new B();
            b.readMarshallable(in);
            using.list_.add(b);
        }
    } else {
        assert size == -1;
        using.list_ = null;
    }
    return using;
}
 

@Test
public void readDoesNotRethrow() {

    // given
    FailedItemMarshaller failedItemMarshaller = new FailedItemMarshaller();

    Bytes bytes = mock(Bytes.class);
    RuntimeException expectedException = spy(new RuntimeException("test exception"));
    when(bytes.inputStream()).thenThrow(expectedException);

    // when
    failedItemMarshaller.read(bytes, null);

    // then
    verify(expectedException, times(1)).getMessage();

}
 

private static long readMessage(Bytes<?> bytes) {
    Jvm.safepoint();
    long start = bytes.readLong();
    long rp = bytes.readPosition();
    long rl = bytes.readLimit();
    long addr = bytes.addressForRead(rp);
    long addrEnd = bytes.addressForRead(rl);
    Memory memory = OS.memory();
    for (addr += 8; addr + 7 < addrEnd; addr += 8)
        memory.readLong(addr);
    Jvm.safepoint();
    return start;
}
 

/**
 * The wire associated with the current index, calling this method moves on the index
 *
 * @return the wire generated by the {@code wireFunction} and populated with the {@code bytes}
 */
@Override
public WireIn wire() {
    if (index == -1) {
        index = statelessRawBytesTailer.lastWrittenIndex();
    }

    final Bytes bytes = statelessRawBytesTailer.readExcept(index);
    index++;
    return wireFunction.apply(bytes);
}
 

/**
 * {@inheritDoc}
 */
@Override
public void sendHeartBeat(final Bytes out, final SessionDetailsProvider sessionDetails) {
    throwExceptionIfClosed();

    delegate.sendHeartBeat(out, sessionDetails);
}
 

@Override
    public void ensureCapacity(long capacity) {
        long cap = lookup.sizeFor(capacity);
        if (cap > bytes.realCapacity()) {
//            BytesStore bytes2 = NativeBytesStore.lazyNativeBytesStoreWithFixedCapacity(Maths.divideRoundUp(cap, OS.pageSize()));
            BytesStore bytes2 = Bytes.wrapForRead(ByteBuffer.allocateDirect(Math.toIntExact(lookup.sizeFor(capacity))));
            bytes2.write(0, bytes);
            bytes.release();
            bytes = bytes2;
        }
    }
 

@Override
public void run() {
    try {
        for (; ; ) {
            if (Thread.currentThread().isInterrupted())
                return;

            Bytes value;

            do {
                value = q.poll(this);
            } while (value == null);

            compresser.setInput(input, (int) value.position(), (int) value.remaining());
            compresser.finish();

            int limit = compresser.deflate(output);
            compresser.end();

            outputBuffer.position(0);
            outputBuffer.limit(limit);

            chronicleQueue.appendDocument(outputBuffer);
        }
    } catch (Exception e) {
        LOG.error("", e);
    }
}
 

@NotNull
@Override
public final T read(@NotNull Bytes in, long size, @Nullable T using) {
    if (using == null)
        using = createInstance();
    using.bytesStore(in.bytesStore(), in.readPosition(), size);
    return using;
}
 

@Override
public void write(Bytes bytes, @NotNull ValueWrapper vw) {
    bytes.writeInt(vw.values.length);

    for (int i = 0; i < vw.values.length; i++) {
        bytes.writeDouble(vw.values[i]);
    }
}
 

public SingleChronicleQueueBuilder codingSuppliers(@Nullable
                                                           Supplier<BiConsumer<BytesStore, Bytes>> encodingSupplier,
                                                   @Nullable Supplier<BiConsumer<BytesStore, Bytes>> decodingSupplier) {
    if ((encodingSupplier == null) != (decodingSupplier == null))
        throw new UnsupportedOperationException("Both encodingSupplier and decodingSupplier must be set or neither");
    this.encodingSupplier = encodingSupplier;
    this.decodingSupplier = decodingSupplier;
    return this;
}
 

private long parseInteger(Bytes s, long off, long len) {
    long lim = off + len;
    boolean neg = false;
    int cutLim = (int) ((Long.MAX_VALUE) % 10);
    int first;
    long ret = (first = s.readByte(off)) - '0';
    if (ret < 0) {
        assert first == '-';
        neg = true;
        cutLim += 1;
        off++;
        ret = s.readByte(off) - '0';

    } else if (ret == 0) {
        assert len == 1;
        return 0;

    } else {
        assert ret > 0 && ret <= 9;
    }
    off++;
    ret = -ret;
    long cutoff = (-Long.MAX_VALUE) / 10;
    while (off < lim) {
        int c = s.readByte(off++) - '0';
        assert(0 <= c && c <= 9);
        if (ret < cutoff || (ret == cutoff && c > cutLim)) {
            throw new NumberFormatException();
        }
        ret = 10 * ret - c;
    }
    return !neg ? -ret : ret;
}
 

public static int size(final BytesMarshallable message) {
    final Bytes<ByteBuffer> buffer = Bytes.elasticByteBuffer();
    try {
        message.writeMarshallable(buffer);
        return (int) buffer.writePosition();
    } finally {
        buffer.releaseLast();
    }
}
 

static void checkPseudoRandom(Bytes bytes, long size) throws java.nio.BufferUnderflowException {
    bytes.readPositionRemaining(0, size);
    int count = 0;
    while (bytes.readRemaining() > 7) {
        count += Long.bitCount(bytes.readLong());
    }
    assertEquals(size * 4, count, size);
}
 

@Override
public void write(Bytes out, V toWrite) {
    try {
        objectMapper.writeValue(out.writer(), toWrite);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

protected void checkWires(final Bytes<?> in, Bytes<?> out, @NotNull final WireType wireType) {
    if (recreateWire) {
        recreateWire = false;
        initialiseInWire(wireType, in);
        initialiseOutWire(out, wireType);
        return;
    }

    if (inWire == null) {
        initialiseInWire(wireType, in);
        recreateWire = false;
    }

    assert inWire.startUse();
    if (inWire.bytes() != in) {
        initialiseInWire(wireType, in);
        recreateWire = false;
    }
    assert inWire.endUse();

    boolean replace = outWire == null;
    if (!replace) {
        assert outWire.startUse();
        replace = outWire.bytes() != out;
        assert outWire.endUse();
    }
    if (replace) {
        initialiseOutWire(out, wireType);
        recreateWire = false;
    }
}
 

@Override
    public void process(@NotNull final Bytes in, @NotNull final Bytes out, T nc) {
//        System.out.println(in.readRemaining());
        if (in.readRemaining() == 0)
            return;
//        System.out.println("P start " + in.toDebugString());
        long toWrite = Math.min(in.readRemaining(), out.writeRemaining());
        out.write(in, in.readPosition(), toWrite);
        in.readSkip(toWrite);
//        System.out.println("... P End " + in.toDebugString());
    }
 

private void resizeInWire(final long size) {
    @NotNull final Bytes<?> bytes = inWire.bytes();
    if (size > bytes.realCapacity()) {
        if (Jvm.isDebugEnabled(getClass()))
            Jvm.debug().on(getClass(), Integer.toHexString(System.identityHashCode(bytes)) + " resized to: " + size);
        bytes.ensureCapacity(size);
    }
}
 

@Test
public void signAndVerify() {
    Bytes privateKeyBuffer = null;
    Bytes secretKeyBuffer = null;
    Bytes privateOrSecret = bft.fromHex(privateOrSecretKey);
    if (privateOrSecret.readRemaining() == Ed25519.SECRET_KEY_LENGTH) {
        secretKeyBuffer = privateOrSecret;
    } else {
        privateKeyBuffer = privateOrSecret;
    }

    Bytes publicKeyBuffer = bft.fromHex(publicKey);
    if (secretKeyBuffer == null) {
        secretKeyBuffer = bft.bytesWithZeros(Ed25519.SECRET_KEY_LENGTH);
        Bytes tmpPublicKeyBuffer = bft.bytesWithZeros(Ed25519.PUBLIC_KEY_LENGTH);
        Ed25519.privateToPublicAndSecret(tmpPublicKeyBuffer, secretKeyBuffer, privateKeyBuffer);
        assertEquals(publicKeyBuffer.toHexString(), tmpPublicKeyBuffer.toHexString());
    }
    Bytes messageBuffer = bft.fromHex(message);
    Bytes signExpectedBuffer;
    if (signExpected.length() == 128) {
        signExpectedBuffer = Bytes.wrapForRead(DatatypeConverter.parseHexBinary(signExpected + message));
    } else {
        signExpectedBuffer = Bytes.wrapForRead(DatatypeConverter.parseHexBinary(signExpected));
    }
    Bytes signedMsgBuffer = bft.fromHex(Ed25519.SIGNATURE_LENGTH, message);
    signedMsgBuffer.writePosition(0);
    Ed25519.sign(signedMsgBuffer, messageBuffer, secretKeyBuffer);
    assertEquals(signExpectedBuffer.toHexString(), signedMsgBuffer.toHexString());
    signedMsgBuffer.readPosition(0);
    publicKeyBuffer.readPositionRemaining(0, Ed25519.PUBLIC_KEY_LENGTH);
    assertTrue(Ed25519.verify(signedMsgBuffer, publicKeyBuffer));
}
 

@NotNull
private Wire readAnywhere(@NotNull final Wire wire) {
    final Bytes<?> bytes = wire.bytes();
    bytes.readLimitToCapacity();
    if (store.dataVersion() > 0)
        wire.usePadding(true);
    return wire;
}
 

private void ensureCapacity() {
    @NotNull final Bytes<?> bytes = inWire.bytes();
    if (bytes.readRemaining() >= 4) {
        final long pos = bytes.readPosition();
        int length = bytes.readInt(pos);
        final long size = pos + Wires.SPB_HEADER_SIZE * 2 + Wires.lengthOf(length);
        if (size > bytes.realCapacity()) {
            resizeInWire(size);
        }
    }
}
 

private void readSome(DirectChronicleQueue chronicle) {
    NativeBytesStore allocate = NativeBytesStore.nativeStoreWithFixedCapacity(EXPECTED_BYTES.length);
    final Bytes toRead = allocate.bytes();
    AtomicLong offset = new AtomicLong(chronicle.firstBytes());
    for (int i = 0; i < RUNS; i++) {
        toRead.clear();
        chronicle.readDocument(offset, toRead);
    }
}