io.protostuff.ByteString Java Examples

@Test
public void testStore() throws Exception {
  try(final LegacyKVStoreProvider kvstore =
    new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false).asLegacy()) {
    kvstore.start();
    ConfigurationStore store = new ConfigurationStore(kvstore);

    ConfigurationEntry supportEntry = new ConfigurationEntry();
    supportEntry.setType("mytype");
    supportEntry.setValue(ByteString.copyFrom("test string", "UTF8"));
    store.put("key", supportEntry);

    ConfigurationEntry retrieved = store.get("key");
    Assert.assertEquals(retrieved.getType(), supportEntry.getType());
    Assert.assertEquals(retrieved.getValue(), supportEntry.getValue());

    store.delete("key");
    retrieved = store.get("key");
    Assert.assertEquals(retrieved, null);
  }
}
 

@JsonCreator
public WriterOptions(
  @JsonProperty("ringCount") Integer ringCount,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("sortColumns") List<String> sortColumns,
  @JsonProperty("distributionColumns") List<String> distributionColumns,
  @JsonProperty("partitionDistributionStrategy") PartitionDistributionStrategy partitionDistributionStrategy,
  @JsonProperty("singleWriter") boolean singleWriter,
  @JsonProperty("recordLimit") long recordLimit,
  @JsonProperty("icebergWriterOperation") IcebergWriterOperation icebergWriterOperation,
  @JsonProperty("extendedProperty") ByteString extendedProperty
  ) {
  this.ringCount = ringCount;
  this.partitionColumns = partitionColumns;
  this.sortColumns = sortColumns;
  this.distributionColumns = distributionColumns;
  this.partitionDistributionStrategy = partitionDistributionStrategy;
  this.singleWriter = singleWriter;
  this.recordLimit = recordLimit;
  this.icebergWriterOperation = icebergWriterOperation;
  this.extendedProperty = extendedProperty;
}
 

@JsonIgnore
@SuppressWarnings({ "rawtypes", "unchecked" })
public FileConfig asFileConfig() {
  buffer.clear();
  FileConfig fc = new FileConfig();
  fc.setType(getFileType());
  fc.setName(name);
  fc.setOwner(owner);
  fc.setCtime(ctime);
  fc.setType(getFileType());
  fc.setTag(getVersion());
  fc.setLocation(location);
  byte[] bytes = ProtobufIOUtil.toByteArray(this, (Schema) getPrivateSchema(), buffer);
  fc.setExtendedConfig(ByteString.copyFrom(bytes));
  fc.setFullPathList(fullPath);
  return fc;
}
 

private <T extends MetadataOption> void addDatasetOptions(Class<T> clazz, DatasetConfig datasetConfig, List<T> outOptions) {
  if(datasetConfig == null) {
    return;
  }

  List<MetadataOption> options = new ArrayList<>();
  if(datasetConfig.getPhysicalDataset() != null && datasetConfig.getPhysicalDataset().getFormatSettings() != null) {
    options.add(new FileConfigOption(datasetConfig.getPhysicalDataset().getFormatSettings()));
  }

  if(datasetConfig.getReadDefinition() != null && datasetConfig.getReadDefinition().getSortColumnsList() != null) {
    options.add(new SortColumnsOption(datasetConfig.getReadDefinition().getSortColumnsList()));
  }

  BatchSchema schema = DatasetHelper.getSchemaBytes(datasetConfig) != null ? CalciteArrowHelper.fromDataset(datasetConfig) : null;
  if(schema != null) {
    options.add(new CurrentSchemaOption(schema));
  }

  if (datasetConfig.getReadDefinition() != null && datasetConfig.getReadDefinition().getExtendedProperty() != null) {
    options.add(new ExtendedPropertyOption(os -> ByteString.writeTo(os, datasetConfig.getReadDefinition().getExtendedProperty())));
  }

  List<T> addOptions = options.stream().filter(o -> clazz.isAssignableFrom(o.getClass())).map(o -> clazz.cast(o)).collect(Collectors.toList());
  outOptions.addAll(addOptions);
}
 

@Test
public void testNASFileStore() throws Exception {

  final CatalogServiceImpl catalog = (CatalogServiceImpl) l(CatalogService.class);
  final SourceConfig config = catalog.getManagedSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME).getId().getClonedConfig();
  final ByteString oldConfig = config.getConfig();
  final HomeFileConf nasHomeFileStore = new HomeFileConf(Path.of("file:///" + BaseTestServer.folder1.getRoot().toString() + "/" + "testNASFileStore/").toString(), "localhost");
  nasHomeFileStore.getFilesystemAndCreatePaths("localhost");
  config.setConnectionConf(nasHomeFileStore);
  catalog.getSystemUserCatalog().updateSource(config);
  HomeFileTool tool = l(HomeFileTool.class);

  try {
    runTests(nasHomeFileStore);
  } finally {
    tool.clear();
    // reset plugin
    SourceConfig backConfig = catalog.getManagedSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME).getId().getClonedConfig();
    backConfig.setConfig(oldConfig);
    catalog.getSystemUserCatalog().updateSource(backConfig);
  }
}
 

public static List<org.apache.arrow.vector.types.pojo.Field> getArrowFieldsFromDatasetConfig(DatasetConfig datasetConfig) {
  List<org.apache.arrow.vector.types.pojo.Field> fields = Lists.newArrayList();
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig);
  if (schemaBytes == null) {
    return null;
  }

  final BatchSchema batchSchema = BatchSchema.deserialize(schemaBytes);
  for (int i = 0; i < batchSchema.getFieldCount(); i++) {
    final org.apache.arrow.vector.types.pojo.Field field = batchSchema.getColumn(i);
    if (!NamespaceTable.SYSTEM_COLUMNS.contains(field.getName())) {
      fields.add(field);
    }
  }

  return fields;
}
 

public ParquetSubScan(
  OpProps props,
  FileConfig formatSettings,
  List<SplitAndPartitionInfo> splits,
  BatchSchema fullSchema,
  List<List<String>> tablePath,
  List<ParquetFilterCondition> conditions,
  StoragePluginId pluginId,
  List<SchemaPath> columns,
  List<String> partitionColumns,
  List<GlobalDictionaryFieldInfo> globalDictionaryEncodedColumns,
  ByteString extendedProperty
) {
  super(props, fullSchema, tablePath, columns);
  this.formatSettings = formatSettings;
  this.splits = splits;
  this.tablePath = tablePath;
  this.conditions = conditions == null ? null : ImmutableList.copyOf(conditions);
  this.pluginId = pluginId;
  this.partitionColumns = partitionColumns;
  this.globalDictionaryEncodedColumns = globalDictionaryEncodedColumns;
  this.extendedProperty = extendedProperty;
}
 

@JsonCreator
public ParquetSubScan(
  @JsonProperty("props") OpProps props,
  @JsonProperty("formatSettings") FileConfig formatSettings,
  @JsonProperty("schema") BatchSchema fullSchema,
  @JsonProperty("referencedTables") List<List<String>> tablePath,
  @JsonProperty("conditions") List<ParquetFilterCondition> conditions,
  @JsonProperty("pluginId") StoragePluginId pluginId,
  @JsonProperty("columns") List<SchemaPath> columns,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("globalDictionaryEncodedColumns") List<GlobalDictionaryFieldInfo> globalDictionaryEncodedColumns,
  @JsonProperty("extendedProperty") ByteString extendedProperty) {

  this(props, formatSettings, null, fullSchema, tablePath, conditions, pluginId, columns, partitionColumns,
    globalDictionaryEncodedColumns, extendedProperty);
}
 

public static String[] getColumnsLowerCase(DatasetConfig datasetConfig) {
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig);
  if (schemaBytes != null) {
    Schema schema = Schema.getRootAsSchema(schemaBytes.asReadOnlyByteBuffer());
    org.apache.arrow.vector.types.pojo.Schema s = org.apache.arrow.vector.types.pojo.Schema.convertSchema(schema);
    return s.getFields().stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new);
  } else {
    // If virtual dataset was created with view fields
    if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) {
      final List<ViewFieldType> viewFieldTypes = datasetConfig.getVirtualDataset().getSqlFieldsList();
      if (notEmpty(viewFieldTypes)) {
        return viewFieldTypes.stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new);
      }
    }
  }

  return new String[0];
}
 

@JsonCreator
public EasySubScan(
  @JsonProperty("props") OpProps props,
  @JsonProperty("settings") FileConfig config,
  @JsonProperty("fullSchema") BatchSchema fullSchema,
  @JsonProperty("tableSchemaPath") List<String> tablePath,
  @JsonProperty("pluginId") StoragePluginId pluginId,
  @JsonProperty("columns") List<SchemaPath> columns,
  @JsonProperty("partitionColumns") List<String> partitionColumns,
  @JsonProperty("extendedProperty") ByteString extendedProperty) {

  this(props, config, null, fullSchema, tablePath, pluginId, columns, partitionColumns, extendedProperty);
}
 

private boolean schemaChanged(MaterializationDescriptor old, Materialization materialization) {
  if (namespaceService == null) {
    return false;
  }
  try {
    //TODO is this enough ? shouldn't we use the dataset hash instead ??
    final NamespaceKey matKey = new NamespaceKey(ReflectionUtils.getMaterializationPath(materialization));
    ByteString schemaString = namespaceService.getDataset(matKey).getRecordSchema();
    BatchSchema newSchema = BatchSchema.deserialize(schemaString);
    BatchSchema oldSchema = ((CachedMaterializationDescriptor) old).getMaterialization().getSchema();
    return !oldSchema.equals(newSchema);
  } catch (NamespaceException e) {
    return true;
  }
}
 

public Bar(int someInt, String someString, Baz baz, Status someEnum,
        ByteString someBytes, boolean someBoolean, float someFloat,
        double someDouble, long someLong)
{
    this.someInt = someInt;
    this.someString = someString;
    this.someBaz = baz;
    this.someEnum = someEnum;
    this.someBytes = someBytes;
    this.someBoolean = someBoolean;
    this.someFloat = someFloat;
    this.someDouble = someDouble;
    this.someLong = someLong;
}
 

private void initIcebergColumnIDList(ByteString extendedProperty) {
  try {
    IcebergProtobuf.IcebergDatasetXAttr icebergDatasetXAttr = LegacyProtobufSerializer.parseFrom(IcebergProtobuf.IcebergDatasetXAttr.PARSER,
      extendedProperty.toByteArray());
    List<IcebergProtobuf.IcebergSchemaField> icebergColumnIDs = icebergDatasetXAttr.getColumnIdsList();
    Map<String, Integer> icebergColumns = new HashMap<>();
    icebergColumnIDs.forEach(field -> icebergColumns.put(field.getSchemaPath(), field.getId()));
    this.icebergColumnIDMap = CaseInsensitiveImmutableBiMap.newImmutableMap(icebergColumns);
  } catch (InvalidProtocolBufferException e) {
    throw new RuntimeException("Could not deserialize Parquet dataset info", e);
  }
}
 

public void testNullMid() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { true, null, false },
            new Character[] { 'a', null, 'b' },
            new Short[] { 1, null, 2 },
            new Integer[] { 1, null, 2 },
            new Long[] { 1l, null, 2l },
            new Float[] { 1.1f, null, 2.2f },
            new Double[] { 1.1d, null, 2.2d },
            new String[] { "a", null, "b" },
            new ByteString[] { ByteString.copyFromUtf8("a"), null, ByteString.copyFromUtf8("b") },
            new byte[][] { new byte[] { 'a' }, null, new byte[] { 'b' } },
            new BigDecimal[] { new BigDecimal(1.1d), null, new BigDecimal(2.2d) },
            new BigInteger[] { new BigInteger("1"), null, new BigInteger("2") },
            new Date[] { new Date(), null, new Date() },
            new Size[] { Size.MEDIUM, null, Size.LARGE },
            new SomePojo[] { new SomePojo("a"), null, new SomePojo("b") }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 

/**
 * Get view fields from dataset config as it is in the batch schema (if not present, defaults to reltree's row type)
 * @param config dataset config
 * @return List of view fields
 */
public static List<ViewFieldType> getViewFields(DatasetConfig config) {
  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(config);
  final boolean virtualDataset = config.getType() == DatasetType.VIRTUAL_DATASET;
  if (schemaBytes == null && virtualDataset) {
    return config.getVirtualDataset().getSqlFieldsList();
  }
  if (schemaBytes != null) {
    return getBatchSchemaFields(BatchSchema.deserialize(schemaBytes.toByteArray()));
  }
  return null;
}
 

public static BatchSchema fromDataset(DatasetConfig config){
  try{
    ByteString bytes = DatasetHelper.getSchemaBytes(config);
    if(bytes == null){
      throw new IllegalStateException(String.format("Schema is currently unavailable for dataset %s.", PathUtils.constructFullPath(config.getFullPathList())));
    }
    return BatchSchema.deserialize(bytes);
  }catch(Exception ex){
    throw new IllegalStateException(String.format("Schema for dataset %s is corrupt.", PathUtils.constructFullPath(config.getFullPathList())), ex);
  }
}
 

public void testNullLast() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { true, false, null },
            new Character[] { 'a', 'b', null },
            new Short[] { 1, 2, null },
            new Integer[] { 1, 2, null },
            new Long[] { 1l, 2l, null },
            new Float[] { 1.1f, 2.2f, null },
            new Double[] { 1.1d, 2.2d, null },
            new String[] { "a", "b", null },
            new ByteString[] { ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b"), null },
            new byte[][] { new byte[] { 'a' }, new byte[] { 'b' }, null },
            new BigDecimal[] { new BigDecimal(1.1d), new BigDecimal(2.2d), null },
            new BigInteger[] { new BigInteger("1"), new BigInteger("2"), null },
            new Date[] { new Date(), new Date(), null },
            new Size[] { Size.MEDIUM, Size.LARGE, null },
            new SomePojo[] { new SomePojo("a"), new SomePojo("b"), null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 

public Foo(List<Integer> someInt, List<String> someString,
        List<Bar> someBar, List<EnumSample> someEnum,
        List<ByteString> someBytes, List<Boolean> someBoolean,
        List<Float> someFloat, List<Double> someDouble, List<Long> someLong)
{
    this.someInt = someInt;
    this.someString = someString;
    this.someBar = someBar;
    this.someEnum = someEnum;
    this.someBytes = someBytes;
    this.someBoolean = someBoolean;
    this.someFloat = someFloat;
    this.someDouble = someDouble;
    this.someLong = someLong;
}
 

@Override
public void recordExtraInfo(String name, byte[] bytes) {
  //TODO DX-10977 the reflection manager should rely on its own observer to store this information in a separate store
  if(job.getJobAttempt().getExtraInfoList() == null) {
    job.getJobAttempt().setExtraInfoList(new ArrayList<ExtraInfo>());
  }

  job.getJobAttempt().getExtraInfoList().add(new ExtraInfo()
      .setData(ByteString.copyFrom(bytes))
      .setName(name));
  storeJob(job);
  super.recordExtraInfo(name, bytes);
}
 

public void testNullFirst() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, true, false },
            new Character[] { null, 'a', 'b' },
            new Short[] { null, 1, 2 },
            new Integer[] { null, 1, 2 },
            new Long[] { null, 1l, 2l },
            new Float[] { null, 1.1f, 2.2f },
            new Double[] { null, 1.1d, 2.2d },
            new String[] { null, "a", "b" },
            new ByteString[] { null, ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b") },
            new byte[][] { null, new byte[] { 'a' }, new byte[] { 'b' } },
            new BigDecimal[] { null, new BigDecimal(1.1d), new BigDecimal(2.2d) },
            new BigInteger[] { null, new BigInteger("1"), new BigInteger("2") },
            new Date[] { null, new Date(), new Date() },
            new Size[] { null, Size.MEDIUM, Size.LARGE },
            new SomePojo[] { null, new SomePojo("a"), new SomePojo("b") }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 

public void testNullAll() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, null, null },
            new Character[] { null, null, null },
            new Short[] { null, null, null },
            new Integer[] { null, null, null },
            new Long[] { null, null, null },
            new Float[] { null, null, null },
            new Double[] { null, null, null },
            new String[] { null, null, null },
            new ByteString[] { null, null, null },
            new byte[][] { null, null, null },
            new BigDecimal[] { null, null, null },
            new BigInteger[] { null, null, null },
            new Date[] { null, null, null },
            new Size[] { null, null, null },
            new SomePojo[] { null, null, null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 

/**
 * Test legal upgrade if prior dremio edition is not specified or editions match
 */
@Test
public void testLegalUpgrade() throws Exception {
  final ByteString prevEdition = ByteString.copyFrom("OSS".getBytes());
  final ConfigurationEntry configurationEntry = new ConfigurationEntry();
  configurationEntry.setValue(prevEdition);
  final LegacyKVStoreProvider kvStoreProvider = new LegacyKVStoreProviderAdapter(
    new LocalKVStoreProvider(CLASSPATH_SCAN_RESULT, null, true, false),
    CLASSPATH_SCAN_RESULT);
  kvStoreProvider.start();
  final ConfigurationStore configurationStore = new ConfigurationStore(kvStoreProvider);
  new Upgrade(DACConfig.newConfig(), CLASSPATH_SCAN_RESULT, false).validateUpgrade(kvStoreProvider, "OSS");
  configurationStore.put(SupportService.DREMIO_EDITION, configurationEntry);
  new Upgrade(DACConfig.newConfig(), CLASSPATH_SCAN_RESULT, false).validateUpgrade(kvStoreProvider, "OSS");
}
 

@GET
@Path("acceleration/settings")
@Produces(APPLICATION_JSON)
public AccelerationSettingsDescriptor getAccelerationSettings() throws NamespaceException {
  final DatasetConfig config = namespaceService.getDataset(datasetPath.toNamespaceKey());
  if (config.getType() == DatasetType.VIRTUAL_DATASET) {
    final String msg = String.format("acceleration settings apply only to physical dataset. %s is a virtual dataset",
        datasetPath.toPathString());
    throw new IllegalArgumentException(msg);
  }

  final ReflectionSettings reflectionSettings = reflectionServiceHelper.getReflectionSettings();
  final AccelerationSettings settings = reflectionSettings.getReflectionSettings(datasetPath.toNamespaceKey());
  final AccelerationSettingsDescriptor descriptor = new AccelerationSettingsDescriptor()
    .setAccelerationRefreshPeriod(settings.getRefreshPeriod())
    .setAccelerationGracePeriod(settings.getGracePeriod())
    .setMethod(settings.getMethod())
    .setRefreshField(settings.getRefreshField())
    .setAccelerationNeverExpire(settings.getNeverExpire())
    .setAccelerationNeverRefresh(settings.getNeverRefresh());

  final ByteString schemaBytes = DatasetHelper.getSchemaBytes(config);
  if (schemaBytes != null) {
    final BatchSchema schema = BatchSchema.deserialize(schemaBytes.toByteArray());
    descriptor.setFieldList(FluentIterable
        .from(schema)
        .transform(new Function<Field, String>() {
          @Nullable
          @Override
          public String apply(@Nullable final Field field) {
            return field.getName();
          }
        })
        .toList()
    );
  }
  return descriptor;
}
 

public void testNullFirstAndLast() throws IOException
{
    Schema<PojoWithNonPrimitiveArrays> schema =
            getSchema(PojoWithNonPrimitiveArrays.class);

    Pipe.Schema<PojoWithNonPrimitiveArrays> pipeSchema =
            ((RuntimeSchema<PojoWithNonPrimitiveArrays>) schema).getPipeSchema();

    PojoWithNonPrimitiveArrays p = new PojoWithNonPrimitiveArrays(
            new Boolean[] { null, true, false, null },
            new Character[] { null, 'a', 'b', null },
            new Short[] { null, 1, 2, null },
            new Integer[] { null, 1, 2, null },
            new Long[] { null, 1l, 2l, null },
            new Float[] { null, 1.1f, 2.2f, null },
            new Double[] { null, 1.1d, 2.2d, null },
            new String[] { null, "a", "b", null },
            new ByteString[] { null, ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b"), null },
            new byte[][] { null, new byte[] { 'a' }, new byte[] { 'b' }, null },
            new BigDecimal[] { null, new BigDecimal(1.1d), new BigDecimal(2.2d), null },
            new BigInteger[] { null, new BigInteger("1"), new BigInteger("2"), null },
            new Date[] { null, new Date(), new Date(), null },
            new Size[] { null, Size.MEDIUM, Size.LARGE, null },
            new SomePojo[] { null, new SomePojo("a"), new SomePojo("b"), null }
            );

    byte[] data = toByteArray(p, schema);

    PojoWithNonPrimitiveArrays pFromByteArray = schema.newMessage();
    mergeFrom(data, 0, data.length, pFromByteArray, schema);
    assertEquals(p, pFromByteArray);

    PojoWithNonPrimitiveArrays pFromStream = schema.newMessage();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    mergeFrom(in, pFromStream, schema);
    assertEquals(p, pFromStream);

    roundTrip(p, schema, pipeSchema);
}
 

public ElasticsearchSubScan(
  OpProps props,
  StoragePluginId pluginId,
  ElasticsearchScanSpec spec,
  List<SplitAndPartitionInfo> splits,
  List<SchemaPath> columns,
  List<String> tableSchemaPath,
  BatchSchema fullSchema,
  ByteString extendedProperty){
  super(props, fullSchema, tableSchemaPath, columns);
  this.pluginId = pluginId;
  this.splits = splits;
  this.spec = spec;
  this.extendedProperty = extendedProperty;
}
 

@JsonCreator
public ElasticsearchSubScan(
    @JsonProperty("props") OpProps props,
    @JsonProperty("pluginId") StoragePluginId pluginId,
    @JsonProperty("spec") ElasticsearchScanSpec spec,
    @JsonProperty("columns") List<SchemaPath> columns,
    @JsonProperty("tableSchemaPath") List<String> tableSchemaPath,
    @JsonProperty("fullSchema") BatchSchema fullSchema,
    @JsonProperty("extendedProperty") ByteString extendedProperty) {
  super(props, fullSchema, tableSchemaPath, columns);
  this.pluginId = pluginId;
  this.spec = spec;
  this.extendedProperty = extendedProperty;
}
 

@Test
public void scalarConstructedObject() throws Exception {
    ScalarFieldTestMsg msg = ScalarFieldTestMsg.newBuilder()
            .setDouble(0.1d)
            .setFloat(0.2f)
            .setInt32(3)
            .setInt64(4L)
            .setUnsignedInt32(5)
            .setUnsignedInt64(6L)
            .setSignedInt32(7)
            .setSignedInt64(8L)
            .setFixed32(9)
            .setFixed64(10L)
            .setSignedFixed32(11)
            .setSignedFixed64(12L)
            .setBool(true)
            .setString("string")
            .setBytes(ByteString.copyFrom(new byte[]{1, 2, 3}))
            .build();
    assertEquals(0.1d, msg.getDouble(), 0.000001d);
    assertEquals(0.2f, msg.getFloat(), 0.000001f);
    assertEquals(3, msg.getInt32());
    assertEquals(4L, msg.getInt64());
    assertEquals(5, msg.getUnsignedInt32());
    assertEquals(6L, msg.getUnsignedInt64());
    assertEquals(7, msg.getSignedInt32());
    assertEquals(8L, msg.getSignedInt64());
    assertEquals(9, msg.getFixed32());
    assertEquals(10L, msg.getFixed64());
    assertEquals(11, msg.getSignedFixed32());
    assertEquals(12L, msg.getSignedFixed64());
    assertEquals(true, msg.getBool());
    assertEquals("string", msg.getString());
    assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), msg.getBytes());
}
 

@Override
public Object readFrom(Input input, Object owner) throws IOException
{
    if (ID_ARRAY_LEN != input.readFieldNumber(this))
        throw new ProtostuffException("Corrupt input.");

    final int len = input.readInt32();

    ByteString[] array = new ByteString[len];
    if (input instanceof GraphInput)
    {
        // update the actual reference.
        ((GraphInput) input).updateLast(array, owner);
    }

    for (int i = 0; i < len;)
    {
        switch (input.readFieldNumber(this))
        {
            case ID_ARRAY_DATA:
                array[i++] = input.readBytes();
                break;
            case ID_ARRAY_NULLCOUNT:
                i += input.readUInt32();
                break;
            default:
                throw new ProtostuffException("Corrupt input.");
        }
    }

    if (0 != input.readFieldNumber(this))
        throw new ProtostuffException("Corrupt input.");

    return array;
}
 

@Override
public void writeTo(Output output, Object value) throws IOException
{
    ByteString[] array = (ByteString[]) value;
    output.writeInt32(ID_ARRAY_LEN, array.length, false);

    int nullCount = 0;
    for (int i = 0, len = array.length; i < len; i++)
    {
        ByteString v = array[i];
        if (v != null)
        {
            if (nullCount != 0)
            {
                output.writeUInt32(ID_ARRAY_NULLCOUNT, nullCount, false);
                nullCount = 0;
            }

            output.writeBytes(ID_ARRAY_DATA, v, true);
        }
        else if (allowNullArrayElement)
        {
            nullCount++;
        }
    }

    // if last element is null
    if (nullCount != 0)
        output.writeUInt32(ID_ARRAY_NULLCOUNT, nullCount, false);
}
 

@SuppressWarnings("all")
@Override
public ByteString readBytes() throws IOException {
    try {
        return (ByteString) byteStringWrapMethod.invoke(null, readByteArray());
    } catch (Exception e) {
        ThrowUtil.throwException(e);
    }
    return null; // never get here
}