com.google.protobuf.DynamicMessage Java Examples

@Override 
public Row expandRow (FDBStore store, Session session,
                        FDBStoreData storeData, Schema schema) {
    ensureRowConverter();
    DynamicMessage msg;
    try {
        msg = DynamicMessage.parseFrom(rowConverter.getMessageType(), storeData.rawValue);
    } catch (InvalidProtocolBufferException ex) {
        ProtobufReadException nex = new ProtobufReadException(rowDataConverter.getMessageType().getName(), ex.getMessage());
        nex.initCause(ex);
        throw nex;
    }
    Row row = rowConverter.decode(msg);
    row = overlayBlobData(row.rowType(), row, store, session);
    return row;
}
 

@Override
public void write(Record record) throws IOException, DataGeneratorException {
  if (closed) {
    throw new IOException("generator has been closed");
  }
  DynamicMessage message = ProtobufTypeUtil.sdcFieldToProtobufMsg(
      record,
      descriptor,
      messageTypeToExtensionMap,
      defaultValueMap
  );
  if (isDelimited) {
    message.writeDelimitedTo(outputStream);
  } else {
    message.writeTo(outputStream);
  }
}
 

private void addRepeatedFields(Map<String, Object> row, String key, Object value, List<Object> fieldValue) {
    if (fieldValue.isEmpty()) {
        return;
    }
    List<Object> repeatedNestedFields = new ArrayList<>();
    String columnName = null;
    for (Object f : fieldValue) {
        if (f instanceof DynamicMessage) {
            ColumnMapping nestedMappings = (ColumnMapping) value;
            repeatedNestedFields.add(getMappings((DynamicMessage) f, nestedMappings));
            columnName = getNestedColumnName(nestedMappings);
        } else {
            repeatedNestedFields.add(f);
            columnName = (String) value;
        }
    }
    row.put(columnName, repeatedNestedFields);
}
 

private Map<String, Object> toOptions(
    Map<Descriptors.FieldDescriptor, Object> optionFields, Boolean useFullName) {
  if (optionFields.size() == 0) {
    return null;
  }

  Map<String, Object> options = new HashMap<>();
  optionFields.forEach(
      (k, v) -> {
        if (v.getClass().getName().equals("com.google.protobuf.DynamicMessage")) {
          options.put(k.getFullName(), toOptions((((DynamicMessage) v).getAllFields()), false));
        } else {
          final String fieldName = useFullName ? k.getFullName() : k.getName();
          options.put(fieldName, v);
        }
      });

  return options;
}
 

@Test
public void deserializeUnionWithUnknownFields() throws Descriptors.DescriptorValidationException {
    // Add a field to the union descriptor message so as
    // to make it possible to read an unknown field.
    final Descriptors.Descriptor biggerUnionDescriptor = addFieldToUnionDescriptor("dummy_field", DescriptorProtos.FieldDescriptorProto.Type.TYPE_BOOL);
    final DynamicMessage message1 = DynamicMessage.newBuilder(biggerUnionDescriptor)
            .setField(biggerUnionDescriptor.findFieldByName("_MySimpleRecord"), TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).build())
            .setField(biggerUnionDescriptor.findFieldByName("dummy_field"), Boolean.TRUE)
            .build();
    RecordSerializationException ex1 = assertThrows(RecordSerializationException.class,
            () -> serializer.deserializeUnion(metaData.getUnionDescriptor(), Tuple.from(1066L), message1.toByteArray(), metaData.getVersion()));
    assertThat(ex1.getMessage(), containsString("Could not deserialize union message"));
    assertThat(ex1.getMessage(), containsString("there are unknown fields"));
    assertThat((Collection<?>)ex1.getLogInfo().get("unknownFields"), not(empty()));

    // Remove a field from the union descriptor and set it.
    final Descriptors.Descriptor smallerUnionDescriptor = removeFieldFromUnionDescriptor("_MySimpleRecord");
    final Message message2 = TestRecords1Proto.RecordTypeUnion.newBuilder()
            .setMySimpleRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).build())
            .build();
    RecordSerializationException ex2 = assertThrows(RecordSerializationException.class,
            () -> serializer.deserializeUnion(smallerUnionDescriptor, Tuple.from(1066L), message2.toByteArray(), metaData.getVersion()));
    assertThat(ex2.getMessage(), containsString("Could not deserialize union message"));
    assertThat(ex2.getMessage(), containsString("there are unknown fields"));
    assertThat((Collection<?>)ex2.getLogInfo().get("unknownFields"), not(empty()));
}
 

@Override
void setOnProtoMessage(Message.Builder message, Map map) {
  if (map != null) {
    FieldDescriptor fieldDescriptor = getFieldDescriptor(message);
    List<Message> messageMap = new ArrayList<>();
    map.forEach(
        (k, v) -> {
          DynamicMessage.Builder builder =
              DynamicMessage.newBuilder(fieldDescriptor.getMessageType());
          FieldDescriptor keyFieldDescriptor =
              fieldDescriptor.getMessageType().findFieldByName("key");
          builder.setField(
              keyFieldDescriptor, this.key.convertToProtoValue(keyFieldDescriptor, k));
          FieldDescriptor valueFieldDescriptor =
              fieldDescriptor.getMessageType().findFieldByName("value");
          builder.setField(
              valueFieldDescriptor, value.convertToProtoValue(valueFieldDescriptor, v));
          messageMap.add(builder.build());
        });
    message.setField(fieldDescriptor, messageMap);
  }
}
 

@Test
public void testSdcToProtobufExtensions() throws Exception {
  List<Record> protobufRecords = ProtobufTestUtil.getProtobufRecords();
  ByteArrayOutputStream bOut = new ByteArrayOutputStream();
  BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(bOut);
  for (int i = 0; i < protobufRecords.size(); i++) {
    DynamicMessage dynamicMessage = ProtobufTypeUtil.sdcFieldToProtobufMsg(
      protobufRecords.get(i),
      md,
      typeToExtensionMap,
      defaultValueMap
    );

    dynamicMessage.writeDelimitedTo(bufferedOutputStream);
  }
  bufferedOutputStream.flush();
  bufferedOutputStream.close();
  ProtobufTestUtil.checkProtobufDataExtensions(bOut.toByteArray());
}
 

/**
 * Handle all the logic leading to the decoding of a Protobuf-encoded binary given a schema file path.
 * @param schema  Schema used to decode the binary data
 * @param messageType   Type of Protobuf Message
 * @param encodedData   Encoded data source
 * @return  A JSON representation of the data, contained in a Java String
 * @throws InvalidProtocolBufferException   Thrown when an error occurs during the encoding of the decoded data into JSON
 * @throws Descriptors.DescriptorValidationException    Thrown when the schema is invalid
 * @throws UnknownMessageTypeException  Thrown when the given message type is not contained in the schema
 * @throws MessageDecodingException Thrown when an error occurs during the binary decoding
 * @throws SchemaLoadingException   Thrown when an error occurs while reading the schema file
 */
public static String decodeProtobuf(DynamicSchema schema, String messageType, InputStream encodedData) throws InvalidProtocolBufferException, Descriptors.DescriptorValidationException, UnknownMessageTypeException, MessageDecodingException, SchemaLoadingException {
    Descriptors.Descriptor descriptor;
    DynamicMessage message;

    descriptor = schema.getMessageDescriptor(messageType);

    if (descriptor == null) {
        throw new UnknownMessageTypeException(messageType);
    }

    try {
        message = DynamicMessage.parseFrom(descriptor, encodedData);
    } catch (IOException e) {
        throw new MessageDecodingException(e);
    }

    return JSONMapper.toJSON(message);
}
 

private boolean dumpPlainField(DynamicMessage.Builder builder, IdentifyDescriptor ident,
        DataDstWriterNode.DataDstFieldDescriptor field, boolean isTopLevel) throws ConvException {
    Descriptors.FieldDescriptor fd = (Descriptors.FieldDescriptor) field.getRawDescriptor();
    if (null == fd) {
        // 不需要提示，如果从其他方式解包协议描述的时候可能有可选字段丢失的
        return false;
    }

    if (null == ident) {
        if (ProgramOptions.getInstance().enbleEmptyList) {
            dumpDefault(builder, fd);
        }
        return false;
    }

    DataContainer<String> res = DataSrcImpl.getOurInstance().getValue(ident, "");
    if (null == res || !res.valid) {
        if (field.isRequired() || ProgramOptions.getInstance().enbleEmptyList) {
            dumpDefault(builder, fd);
        }
        return false;
    }

    return dumpPlainField(builder, ident, field, isTopLevel, res.value);
}
 

@Test
public void concatenateSingleMessage() throws Descriptors.DescriptorValidationException {
    long kafkaOffset = 21L;
    DynamicMessage.Builder inMsg = createBodyBuilder();
    Descriptors.Descriptor inMsgDesc = inMsg.getDescriptorForType();

    inMsg.setField(inMsgDesc.findFieldByName("bodyInt"), 1);
    inMsg.setField(inMsgDesc.findFieldByName("bodyString"), "one");

    Map<String, Object> expectedValues = new HashMap<>();
    expectedValues.put("bodyInt", 1);
    expectedValues.put("bodyString", "one");
    expectedValues.put(ProtoConcatenator.TIMESTAMP_FIELD_NAME, 21L);
    expectedValues.put(ProtoConcatenator.KAFKA_OFFSET, kafkaOffset);

    testAllOutTypesWith(21L, Collections.singletonList(inMsg.build()), expectedValues, kafkaOffset);
}
 

@Test
public void concatenateDifferentMessages() throws Descriptors.DescriptorValidationException {
    long kafkaOffset = 0L;
    DynamicMessage.Builder headerMessageBuilder = createHeaderMessageBuilder();
    Descriptors.Descriptor headerMsgDesc = headerMessageBuilder.getDescriptorForType();

    headerMessageBuilder.setField(headerMsgDesc.findFieldByName("id"), 1)
            .setField(headerMsgDesc.findFieldByName("name"), "one");

    DynamicMessage.Builder bodyMessageBuilder = createBodyBuilder();
    Descriptors.Descriptor bodyMsgDesc = bodyMessageBuilder.getDescriptorForType();

    bodyMessageBuilder.setField(bodyMsgDesc.findFieldByName("bodyInt"), 2)
            .setField(bodyMsgDesc.findFieldByName("bodyString"), "two");

    Map<String, Object> expectedValues = new HashMap<>();
    expectedValues.put("id", 1);
    expectedValues.put("name", "one");
    expectedValues.put("bodyInt", 2);
    expectedValues.put("bodyString", "two");
    expectedValues.put(ProtoConcatenator.TIMESTAMP_FIELD_NAME, 0L);
    expectedValues.put(ProtoConcatenator.KAFKA_OFFSET, kafkaOffset);

    testAllOutTypesWith(0L, Arrays.asList(headerMessageBuilder.build(), bodyMessageBuilder.build()), expectedValues, kafkaOffset);
}
 

@Test
public void testProtoToSdcExtensionFields() throws Exception {

  List<DynamicMessage> messages = ProtobufTestUtil.getMessages(
    md,
    extensionRegistry,
    ProtobufTestUtil.getProtoBufData()
  );

  for (int i = 0; i < messages.size(); i++) {
    DynamicMessage m = messages.get(i);
    Record record = RecordCreator.create();
    Field field = ProtobufTypeUtil.protobufToSdcField(record, "", md, typeToExtensionMap, m);
    Assert.assertNotNull(field);
    ProtobufTestUtil.checkProtobufRecordsForExtensions(field, i);
  }
}
 

@Nonnull
@Override
public byte[] serialize(@Nonnull RecordMetaData metaData,
                        @Nonnull RecordType recordType,
                        @Nonnull Message record,
                        @Nullable StoreTimer timer) {
    long startTime = System.nanoTime();
    try {
        // Wrap in union message, if needed.
        Message storedRecord = record;
        Descriptors.Descriptor unionDescriptor = metaData.getUnionDescriptor();
        if (unionDescriptor != null) {
            DynamicMessage.Builder unionBuilder = DynamicMessage.newBuilder(unionDescriptor);
            Descriptors.FieldDescriptor unionField = metaData.getUnionFieldForRecordType(recordType);
            unionBuilder.setField(unionField, record);
            storedRecord = unionBuilder.build();
        }
        return serializeToBytes(storedRecord);
    } finally {
        if (timer != null) {
            timer.recordSinceNanoTime(Events.SERIALIZE_PROTOBUF_RECORD, startTime);
        }
    }
}
 

@Nonnull
@Override
@SpotBugsSuppressWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
public Message deserialize(@Nonnull final RecordMetaData metaData,
                           @Nonnull final Tuple primaryKey,
                           @Nonnull final byte[] serialized,
                           @Nullable StoreTimer timer) {
    final long startTime = System.nanoTime();
    try {
        final Descriptors.Descriptor unionDescriptor = metaData.getUnionDescriptor();
        final DynamicMessage unionMessage = deserializeUnion(unionDescriptor, primaryKey, serialized, metaData.getVersion());
        return getUnionField(unionMessage, primaryKey).getRight();
    } finally {
        if (timer != null) {
            timer.recordSinceNanoTime(Events.DESERIALIZE_PROTOBUF_RECORD, startTime);
        }
    }
}
 

public ProtobufDataParser(
    ProtoConfigurableEntity.Context context,
    String messageId,
    Descriptors.Descriptor descriptor,
    Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
    ExtensionRegistry extensionRegistry,
    InputStream inputStream,
    String readerOffset,
    int maxObjectLength,
    boolean isDelimited
) throws IOException, Descriptors.DescriptorValidationException, DataParserException {
  this.context = context;
  this.inputStream = new OverrunInputStream(inputStream, maxObjectLength, true);
  this.messageId = messageId;
  this.messageTypeToExtensionMap = messageTypeToExtensionMap;
  this.extensionRegistry = extensionRegistry;
  this.descriptor = descriptor;
  this.builder = DynamicMessage.newBuilder(descriptor);
  this.isDelimited = isDelimited;

  // skip to the required location
  if (readerOffset != null && !readerOffset.isEmpty() && !readerOffset.equals("0")) {
    int offset = Integer.parseInt(readerOffset);
    this.inputStream.skip(offset);
  }
}
 

@Test
public void testNonEmptyRepeated() throws DataGeneratorException {
  Record r = RecordCreator.create();
  Map<String, Field> repeated = new HashMap<>();
  repeated.put(
      "samples",
      Field.create(
          Field.Type.LIST,
          Arrays.asList(
              Field.create(1),
              Field.create(2),
              Field.create(3),
              Field.create(4),
              Field.create(5)
          )
      )
  );
  r.set(Field.create(repeated));
  Descriptors.Descriptor descriptor = RepeatedProto.getDescriptor().findMessageTypeByName("Repeated");
  // repeated field samples is null and ignored
  DynamicMessage dynamicMessage = ProtobufTypeUtil.sdcFieldToProtobufMsg(
    r,
    descriptor,
    typeToExtensionMap,
    defaultValueMap
  );
  // null repeated fields are treated as empty arrays
  Object samples = dynamicMessage.getField(descriptor.findFieldByName("samples"));
  Assert.assertNotNull(samples);
  Assert.assertTrue(samples instanceof List);
  Assert.assertEquals(5, ((List)samples).size());
}
 

@Test
public void testDynamicMessage() throws Exception {
  DynamicMessage message =
      DynamicMessage.newBuilder(MessageA.getDescriptor())
          .setField(
              MessageA.getDescriptor().findFieldByNumber(MessageA.FIELD1_FIELD_NUMBER), "foo")
          .build();
  Coder<DynamicMessage> coder = DynamicProtoCoder.of(message.getDescriptorForType());

  // Special code to check the DynamicMessage equality (@see IsDynamicMessageEqual)
  for (Coder.Context context : ALL_CONTEXTS) {
    CoderProperties.coderDecodeEncodeInContext(
        coder, context, message, IsDynamicMessageEqual.equalTo(message));
  }
}
 

static Message newOuter(Message value1, int value2, Message... value3) {
    FieldDescriptor field1 = OUTER_TYPE.getFields().get(0);
    FieldDescriptor field2 = OUTER_TYPE.getFields().get(1);
    FieldDescriptor field3 = OUTER_TYPE.getFields().get(2);
    return DynamicMessage.newBuilder(OUTER_TYPE)
            .setField(field1, value1)
            .setField(field2, value2)
            .setField(field3, Arrays.asList(value3))
            .build();
}
 

@Test
public void testNullRepeatedRowToProto() {
  ProtoDynamicMessageSchema schemaProvider =
      schemaFromDescriptor(RepeatPrimitive.getDescriptor());
  SerializableFunction<Row, DynamicMessage> fromRow = schemaProvider.getFromRowFunction();
  assertEquals(NULL_REPEATED_PROTO.toString(), fromRow.apply(NULL_REPEATED_ROW).toString());
}
 

@Override
public T apply(Row input) {
  DynamicMessage.Builder builder = context.invokeNewBuilder();
  Iterator values = input.getValues().iterator();
  Iterator<Convert> convertIterator = converters.iterator();

  for (int i = 0; i < input.getValues().size(); i++) {
    Convert convert = convertIterator.next();
    Object value = values.next();
    convert.setOnProtoMessage(builder, value);
  }
  return (T) builder.build();
}
 

private static DynamicMessage.Builder createHeaderMessageBuilder() throws Descriptors.DescriptorValidationException {
    String messageName = "Header";
    DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder();

    MessageDefinition msgDef = MessageDefinition.newBuilder(messageName)
            .addField("required", "int32", "id", 1)
            .addField("optional", "string", "name", 2)
            .build();

    schemaBuilder.addMessageDefinition(msgDef);
    DynamicSchema schema = schemaBuilder.build();

    return schema.newMessageBuilder(messageName);
}
 

private static DynamicMessage createEmptyMessage() throws Descriptors.DescriptorValidationException {
    String messageName = "Empty";
    DynamicSchema.Builder builder = DynamicSchema.newBuilder();

    builder.addMessageDefinition(MessageDefinition.newBuilder(messageName).build());

    return builder.build().newMessageBuilder(messageName).build();
}
 

private io.grpc.MethodDescriptor<DynamicMessage, DynamicMessage> createGrpcMethodDescriptor(MethodDescriptor descriptor) {
    return io.grpc.MethodDescriptor.<DynamicMessage, DynamicMessage>newBuilder()
            .setType(fetchMethodType(descriptor))
            .setFullMethodName(fetchFullMethodName(descriptor))
            .setRequestMarshaller(new DynamicMessageMarshaller(descriptor.getInputType()))
            .setResponseMarshaller(new DynamicMessageMarshaller(descriptor.getOutputType()))
            .build();
}
 

@Test
public void testDynamicNestedRepeatedMessage() throws Exception {
  DynamicMessage message =
      DynamicMessage.newBuilder(MessageA.getDescriptor())
          .setField(
              MessageA.getDescriptor().findFieldByNumber(MessageA.FIELD1_FIELD_NUMBER), "foo")
          .addRepeatedField(
              MessageA.getDescriptor().findFieldByNumber(MessageA.FIELD2_FIELD_NUMBER),
              DynamicMessage.newBuilder(MessageB.getDescriptor())
                  .setField(
                      MessageB.getDescriptor().findFieldByNumber(MessageB.FIELD1_FIELD_NUMBER),
                      true)
                  .build())
          .addRepeatedField(
              MessageA.getDescriptor().findFieldByNumber(MessageA.FIELD2_FIELD_NUMBER),
              DynamicMessage.newBuilder(MessageB.getDescriptor())
                  .setField(
                      MessageB.getDescriptor().findFieldByNumber(MessageB.FIELD1_FIELD_NUMBER),
                      false)
                  .build())
          .build();
  Coder<DynamicMessage> coder = DynamicProtoCoder.of(message.getDescriptorForType());

  // Special code to check the DynamicMessage equality (@see IsDynamicMessageEqual)
  for (Coder.Context context : ALL_CONTEXTS) {
    CoderProperties.coderDecodeEncodeInContext(
        coder, context, message, IsDynamicMessageEqual.equalTo(message));
  }
}
 

@Test
public void testRepeatedRowToProto() {
  ProtoDynamicMessageSchema schemaProvider =
      schemaFromDescriptor(RepeatPrimitive.getDescriptor());
  SerializableFunction<Row, DynamicMessage> fromRow = schemaProvider.getFromRowFunction();
  assertEquals(REPEATED_PROTO.toString(), fromRow.apply(REPEATED_ROW).toString());
}
 

private Map<String, Object> getMappings(DynamicMessage message, ColumnMapping columnMapping) {
    if (message == null || columnMapping == null || columnMapping.isEmpty()) {
        return Collections.emptyMap();
    }
    Descriptors.Descriptor descriptorForType = message.getDescriptorForType();

    Map<String, Object> row = new HashMap<>(columnMapping.size());
    columnMapping.forEach((key, value) -> {
        String columnName = value.toString();
        String column = key.toString();
        if (column.equals(Config.RECORD_NAME)) {
            return;
        }
        Integer protoIndex = Integer.valueOf(column);
        Descriptors.FieldDescriptor fieldDesc = descriptorForType.findFieldByNumber(protoIndex);
        if (fieldDesc != null && !message.getField(fieldDesc).toString().isEmpty()) {
            Object field = message.getField(fieldDesc);
            ProtoField protoField = FieldFactory.getField(fieldDesc, field);
            Object fieldValue = protoField.getValue();

            if (fieldValue instanceof List) {
                addRepeatedFields(row, (String) key, value, (List<Object>) fieldValue);
                return;
            }

            if (protoField.getClass().getName().equals(NestedField.class.getName())) {
                try {
                    columnName = getNestedColumnName((ColumnMapping) value);
                    fieldValue = getMappings((DynamicMessage) field, (ColumnMapping) value);
                } catch (Exception e) {
                    log.error("Exception::Handling nested field failure: {}", e.getMessage());
                }
            }
            row.put(columnName, fieldValue);
        }
    });
    return row;
}
 

private static void setRepeatedField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor dstFieldDescriptor,
                                     Map.Entry<Descriptors.FieldDescriptor, Object> entry) {
    @SuppressWarnings("unchecked")
    final Collection<Object> values = (Collection<Object>) entry.getValue();

    for (Object value : values) {
        builder.addRepeatedField(dstFieldDescriptor, value);
    }
}
 

private boolean dumpPlainField(DynamicMessage.Builder builder, IdentifyDescriptor ident,
        DataDstWriterNode.DataDstOneofDescriptor field, boolean isTopLevel, String input) throws ConvException {
    if (field == null) {
        return false;
    }

    Object[] res = parsePlainDataOneof(input, ident, field);
    if (null == res) {
        return false;
    }

    if (res.length < 1) {
        return false;
    }

    DataDstWriterNode.DataDstFieldDescriptor sub_field = (DataDstWriterNode.DataDstFieldDescriptor) res[0];

    if (sub_field == null) {
        return false;
    }

    if (res.length == 1) {
        dumpDefault(builder, (Descriptors.FieldDescriptor) sub_field.getRawDescriptor());
        return true;
    }

    // 非顶层，不用验证类型
    return dumpPlainField(builder, null, sub_field, false, (String) res[1]);
}
 

private static DynamicMessage dynamic(Message message) {
  try {
    return DynamicMessage.parseFrom(message.getDescriptorForType(), message.toByteString());
  } catch (InvalidProtocolBufferException e) {
    throw new AssertionError(e);
  }
}
 

private static DynamicMessage.Builder createMessageWithRepeatedField()
        throws Descriptors.DescriptorValidationException {
    DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder();

    MessageDefinition msgDef = MessageDefinition.newBuilder("Repeated")
            .addField("repeated", "int32", "repeated_field", 1)
            .build();

    schemaBuilder.addMessageDefinition(msgDef);
    DynamicSchema schema = schemaBuilder.build();

    return schema.newMessageBuilder("Repeated");
}