org.msgpack.core.MessagePacker Java Examples

/**
 * Serializes the {@code messages} into the stream using the given schema.
 */
public static <T> void writeListTo(MessageBufferOutput out, List<T> messages,
        Schema<T> schema, boolean numeric) throws IOException
{

    MessagePacker packer = MessagePack.newDefaultPacker(out);

    try
    {
        writeListTo(packer, messages, schema, numeric);
    }
    finally
    {
        packer.flush();
    }
}
 

/**
 * Serializes the {@code messages} into the stream using the given schema.
 */
public static <T> void writeListTo(OutputStream out, List<T> messages,
        Schema<T> schema, boolean numeric) throws IOException
{

    MessagePacker packer = MessagePack.newDefaultPacker(out);

    try
    {
        writeListTo(packer, messages, schema, numeric);
    }
    finally
    {
        packer.flush();
    }
}
 

private void packArray(Object obj, MessagePacker packer) throws IOException {
    if (BArray.class.isInstance(obj)) {
        BArray array = (BArray) obj;
        packer.packArrayHeader(array.size());
        for (var entry : array) {
            packAny(entry, packer);
        }
        return;
    }

    packer.packArrayHeader(ArrayUtils.length(obj));
    ArrayUtils.foreach(obj, ele -> {
        try {
            packAny(ele, packer);
        } catch (IOException e) {
            throw new RuntimeIOException(e);
        }
    });
}
 

/**
 * Serializes the {@code message} into an {@link MessageBufferOutput} using the given {@code schema}.
 */
public static <T> void writeTo(MessageBufferOutput out, T message, Schema<T> schema, boolean numeric)
        throws IOException
{

    MessagePacker packer = MessagePack.newDefaultPacker(out);

    try
    {
        writeTo(packer, message, schema, numeric);
    }
    finally
    {
        packer.flush();
    }
}
 

@Override
public void setTimestamp(long timestamp)
{
    int mapSize = mapValueBytes.map().size();
    try (ByteArrayOutputStream output = new ByteArrayOutputStream(mapValueBytes.byteArray().length + 16)) {
        try (MessagePacker packer = MessagePack.newDefaultPacker(output)) {
            if (mapValueBytes.map().containsKey(KEY_TIME)) {
                packer.packMapHeader(mapSize);
            }
            else {
                packer.packMapHeader(mapSize + 1);
                KEY_TIME.writeTo(packer);
                packer.packLong(timestamp);
            }

            for (Map.Entry<Value, Value> entry : mapValueBytes.map().entrySet()) {
                packer.packValue(entry.getKey());
                packer.packValue(entry.getValue());
            }
        }
        mapValueBytes = new MapValueBytes(ByteBuffer.wrap(output.toByteArray()));
    }
    catch (IOException e) {
        throw new IllegalStateException("Failed to upsert `time` field", e);
    }
}
 

@Override
public void flush() throws IOException {
    if (rootStackItem != null) {
        if (rootStackItem instanceof StackItemForObject) {
            packObject((StackItemForObject) rootStackItem);
        }
        else if (rootStackItem instanceof StackItemForArray) {
            packArray((StackItemForArray) rootStackItem);
        }
        else {
            throw new IllegalStateException("Unexpected rootStackItem: " + rootStackItem);
        }
        MessagePacker messagePacker = getMessagePacker();
        messagePacker.flush();
    }
}
 

private void packArray(StackItemForArray stackItem) throws IOException {
    List<Object> values = stackItem.getValues();

    MessagePacker messagePacker = getMessagePacker();
    messagePacker.packArrayHeader(values.size());

    for (int i = 0; i < values.size(); i++) {
        Object v = values.get(i);
        packValue(v);
    }
}
 

/**
 * Serializes the {@code messages} into the generator using the given schema.
 */
public static <T> void writeListTo(MessagePacker packer, List<T> messages,
        Schema<T> schema, boolean numeric) throws IOException
{
    MsgpackGenerator generator = new MsgpackGenerator(numeric);
    MsgpackOutput output = new MsgpackOutput(generator, schema);

    for (T m : messages)
    {
        schema.writeTo(output, m);
        generator.writeTo(packer);
        generator.reset();
    }

}
 

public static <T> void writeTo(MessagePacker packer, T message, Schema<T> schema, boolean numeric)
        throws IOException
{
    MsgpackGenerator generator = new MsgpackGenerator(numeric);
    MsgpackOutput output = new MsgpackOutput(generator, schema);
    schema.writeTo(output, message);
    generator.writeTo(packer);
}
 

/**
 * Serializes the {@code message} into an {@link OutputStream} using the given {@code schema}.
 */
public static <T> void writeTo(OutputStream out, T message, Schema<T> schema, boolean numeric) throws IOException
{

    MessagePacker packer = MessagePack.newDefaultPacker(out);

    try
    {
        writeTo(packer, message, schema, numeric);
    }
    finally
    {
        packer.flush();
    }
}
 

@Override
public void writeTo(MessagePacker packer) throws IOException
{
    int size = list.size();
    packer.packArrayHeader(size);
    for (int i = 0; i != size; i++)
    {
        list.get(i).writeTo(packer);
    }
}
 

@Override
public void write(KeyValue keyValue) throws IOException {
    byte[] kafkaKey = keyValue.hasKafkaKey() ? keyValue.getKafkaKey() : new byte[0];
    long timestamp = keyValue.getTimestamp();
    final int timestampLength = (keyValue.hasTimestamp()) ? 10 : 0;
    // output size estimate
    // 1 - map header
    // 1 - message pack key
    // 9 - max kafka offset
    // 1 - message pack key
    // 9 - kafka timestamp
    // 1 - message pack key
    // 5 - max (sane) kafka key size
    // N - size of kafka key
    // = 27 + N
    ByteArrayOutputStream out = new ByteArrayOutputStream(17 + timestampLength + kafkaKey.length);
    MessagePacker packer = MessagePack.newDefaultPacker(out)
            .packMapHeader(numberOfFieldsMappedInHeader(keyValue))
            .packInt(KAFKA_MESSAGE_OFFSET)
            .packLong(keyValue.getOffset());

    if (keyValue.hasTimestamp())
        packer.packInt(KAFKA_MESSAGE_TIMESTAMP)
                .packLong(timestamp);

    if (keyValue.hasKafkaKey())
        packer.packInt(KAFKA_HASH_KEY)
                .packBinaryHeader(kafkaKey.length)
                .writePayload(kafkaKey);

    packer.close();
    byte[] outBytes = out.toByteArray();
    this.mKey.set(outBytes, 0, outBytes.length);
    this.mValue.set(keyValue.getValue(), 0, keyValue.getValue().length);
    this.mWriter.append(this.mKey, this.mValue);
}
 

private MessagePacker getMessagePacker() {
    MessagePacker messagePacker = messagePackersHolder.get();
    if (messagePacker == null) {
        throw new IllegalStateException("messagePacker is null");
    }
    return messagePacker;
}
 

@Override
public void close() throws IOException {
    try {
        flush();
    }
    catch (Exception e) {
        e.printStackTrace();
    }
    finally {
        MessagePacker messagePacker = getMessagePacker();
        messagePacker.close();
    }
}
 

private void packObject(StackItemForObject stackItem) throws IOException {
    List<String> keys = stackItem.getKeys();
    List<Object> values = stackItem.getValues();

    MessagePacker messagePacker = getMessagePacker();
    messagePacker.packMapHeader(keys.size());

    for (int i = 0; i < keys.size(); i++) {
        messagePacker.packString(keys.get(i));
        Object v = values.get(i);
        packValue(v);
    }
}
 

@ParameterizedTest
@MethodSource("sslFlagsProvider")
public void testWithAckResponseWithProperToken(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(2, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                String ackResponseToken = map.get(KEY_OPTION_CHUNK).asRawValue().asString();
                assertNotNull(ackResponseToken);

                MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream());
                packer.packMapHeader(1)
                        .packString("ack").packString(ackResponseToken)
                        .close();

                // Close the input stream after closing the output stream to avoid closing a socket too early
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);
                builder.setAckResponseMode(true);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertNull(exception);
}
 

@ParameterizedTest
@MethodSource("sslFlagsProvider")
void testWithAckResponseButWrongReceiveToken(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(2, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString());

                MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream());
                packer.packMapHeader(1)
                        .packString("ack").packString(UUID.randomUUID().toString())
                        .close();

                // Close the input stream after closing the output stream to avoid closing a socket too early
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);
                builder.setAckResponseMode(true);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertEquals(exception.getClass(), TimeoutException.class);
}
 

@Override
public void serialize( Options options, OutputStream output, @Optional Object object )
{
    try( MessagePacker packer = MessagePack.newDefaultPacker( output ) )
    {
        Value value = doSerialize( options, object, true );
        packer.packValue( value );
        packer.flush();
    }
    catch( IOException ex )
    {
        throw new SerializationException( "Unable to serialize " + object, ex );
    }
}
 

/**
 * Create a new PackedBuffer.
 *
 * @param bb        The ByteBuffer to use to create the packed buffer.
 */
PackedBuffer(ByteBuffer bb) {
  this.bb = bb;
  this.out = new PackedBufferOutput();
  this.temp = new byte[SPAN_ID_BYTE_LENGTH];
  this.packer = new MessagePacker(out, MSGPACK_CONF);
}
 

private void packPojo(Object target, MessagePacker packer) throws IOException {
    PojoGetter.of(target).shallowly(true).walker((indicator, value, signature, proxy) -> {
        try {
            switch (indicator) {
            case KEY_NULL:
                if (!compact) {
                    packer.packString((String) value);
                    packer.packNil();
                }
                break;
            case KEY:
                packer.packString((String) value);
                break;
            case START_MAP:
                packer.packMapHeader((int) value);
                break;
            case VALUE:
                packAny(value, packer);
                break;
            default:
                // do nothing...
                break;
            }
        } catch (Exception e) {
            throw new BeanSerializationException("Error while serialize pojo", e);
        }
    }).walk();
}
 

public Visitor(MessagePacker packer) {
    this.packer = packer;
}
 

private void packValue(Object v) throws IOException {
    MessagePacker messagePacker = getMessagePacker();
    if (v == null) {
        messagePacker.packNil();
    }
    else if (v instanceof Integer) {
        messagePacker.packInt((Integer) v);
    }
    else if (v instanceof ByteBuffer) {
        ByteBuffer bb = (ByteBuffer) v;
        messagePacker.packBinaryHeader(bb.limit());
        messagePacker.writePayload(bb);
    }
    else if (v instanceof String) {
        messagePacker.packString((String) v);
    }
    else if (v instanceof Float) {
        messagePacker.packFloat((Float) v);
    }
    else if (v instanceof Long) {
        messagePacker.packLong((Long) v);
    }
    else if (v instanceof StackItemForObject) {
        packObject((StackItemForObject) v);
    }
    else if (v instanceof StackItemForArray) {
        packArray((StackItemForArray) v);
    }
    else if (v instanceof Double) {
        messagePacker.packDouble((Double) v);
    }
    else if (v instanceof BigInteger) {
        messagePacker.packBigInteger((BigInteger) v);
    }
    else if (v instanceof BigDecimal) {
        // TODO
        throw new NotImplementedException();
    }
    else if (v instanceof Boolean) {
        messagePacker.packBoolean((Boolean) v);
    }
    else {
        throw new IllegalArgumentException(v.toString());
    }
}
 

@Test
public void testParserShouldReadArray() throws IOException {
    MessagePacker packer = new MessagePacker(new OutputStreamBufferOutput(out));
    packer.packArrayHeader(10);
    // #1
    packer.packArrayHeader(3);
    {
        packer.packLong(Long.MAX_VALUE);
        packer.packNil();
        packer.packString("FOO BAR");
    }
    // #2
    packer.packString("str");
    // #3
    packer.packInt(Integer.MAX_VALUE);
    // #4
    packer.packLong(Long.MIN_VALUE);
    // #5
    packer.packFloat(Float.MAX_VALUE);
    // #6
    packer.packDouble(Double.MIN_VALUE);
    // #7
    BigInteger bi = new BigInteger(Long.toString(Long.MAX_VALUE));
    bi = bi.add(BigInteger.ONE);
    packer.packBigInteger(bi);
    // #8
    byte[] bytes = new byte[]{(byte) 0xFF, (byte) 0xFE, 0x01, 0x00};
    packer.packBinaryHeader(bytes.length);
    packer.writePayload(bytes);
    // #9
    packer.packMapHeader(2);
    {
        packer.packString("child_map_name");
        packer.packString("komamitsu");
        packer.packString("child_map_age");
        packer.packInt(42);
    }
    // #10
    packer.packBoolean(true);

    packer.flush();

    bytes = out.toByteArray();

    TypeReference<List<Object>> typeReference = new TypeReference<List<Object>>(){};
    List<Object> array = objectMapper.readValue(bytes, typeReference);
    assertEquals(10, array.size());
    int i = 0;
    // #1
    List<Object> childArray = (List<Object>) array.get(i++);
    {
        int j = 0;
        assertEquals(Long.MAX_VALUE, childArray.get(j++));
        assertEquals(null, childArray.get(j++));
        assertEquals("FOO BAR", childArray.get(j++));
    }
    // #2
    assertEquals("str", array.get(i++));
    // #3
    assertEquals(Integer.MAX_VALUE, array.get(i++));
    // #4
    assertEquals(Long.MIN_VALUE, array.get(i++));
    // #5
    assertEquals(Float.MAX_VALUE, (Double)array.get(i++), 0.001f);
    // #6
    assertEquals(Double.MIN_VALUE, (Double)array.get(i++), 0.001f);
    // #7
    assertEquals(bi, array.get(i++));
    // #8
    byte[] bs = (byte[]) array.get(i++);
    assertEquals(4, bs.length);
    assertEquals((byte)0xFF, bs[0]);
    assertEquals((byte)0xFE, bs[1]);
    assertEquals((byte)0x01, bs[2]);
    assertEquals((byte)0x00, bs[3]);
    // #9
    Map<String, Object> childMap = (Map<String, Object>) array.get(i++);
    {
        assertEquals(2, childMap.keySet().size());
        for (Map.Entry<String, Object> entry : childMap.entrySet()) {
            String k = entry.getKey();
            Object v = entry.getValue();
            if (k.equals("child_map_name")) {
                assertEquals("komamitsu", v);
            }
            else if (k.equals("child_map_age")) {
                assertEquals(42, v);
            }
        }
    }
    // #10
    assertEquals(true, array.get(i++));
}
 

public void writeTo(MessagePacker packer) throws IOException
{
    toValue().writeTo(packer);
}
 

@Override
public void writeTo(MessagePacker pk)
        throws IOException
{
    pk.packFloat(value);
}
 

public MessagePacker reset(OutputStream out) throws IOException {
    this.output.reset(out);
    return this.packer;
}
 

private void packValue(Object obj, MessagePacker packer) throws IOException {
    var value = BValue.of(obj);
    var type = value.getType();
    switch (type) {
    case BOOLEAN:
        packer.packBoolean(value.getBoolean());
        return;
    case BYTE:
        packer.packByte(value.getByte());
        return;
    case CHAR:
    case SHORT:
        packer.packShort(value.getShort());
        return;
    case DOUBLE:
        packer.packDouble(value.getDouble());
        return;
    case FLOAT:
        packer.packFloat(value.getFloat());
        return;
    case INTEGER:
        packer.packInt(value.getInteger());
        return;
    case LONG:
        packer.packLong(value.getLong());
        return;
    case NULL:
        packer.packNil();
        return;
    case RAW:
        byte[] bytes = value.getRaw();
        packer.packBinaryHeader(bytes.length);
        packer.addPayload(bytes);
        return;
    case STRING:
        packer.packString(value.getString());
        return;
    case GENERIC_NUMBER:
        packer.packDouble(value.getDouble());
        return;
    default:
        throw new BeanSerializationException("Cannot writeValue object type: " + type);
    }
}
 

private void packAny(Object obj, MessagePacker packer) throws IOException {
    if (obj == null) {
        packer.packNil();
        return;
    }

    if (BElement.class.isInstance(obj)) {
        BElement element = (BElement) obj;

        if (element.isValue()) {
            packValue(element.asValue(), packer);
            return;
        }

        if (element.isArray()) {
            packArray(element.asArray(), packer);
            return;
        }

        if (element.isObject()) {
            packMap(element.asObject(), packer);
            return;
        }

        if (element.isReference()) {
            packAny(element.asReference().getReference(), packer);
            return;
        }

        throw new BeanSerializationException("Cannot serialize belement which instance of: " + element.getClass());
    }

    Class<?> type;
    if (Collection.class.isInstance(obj) || (type = obj.getClass()).isArray()) {
        packArray(obj, packer);
        return;
    }

    if (Map.class.isInstance(obj)) {
        packMap(obj, packer);
        return;
    }

    if (PrimitiveUtils.isPrimitive(type)) {
        packValue(obj, packer);
        return;
    }

    packPojo(obj, packer);
}
 

public OutputImpl(MessagePacker mp) {
    this.mp = mp;
}