com.google.flatbuffers.FlatBufferBuilder Java Examples

protected int stringBuffer(FlatBufferBuilder builder, DataBuffer buffer) {
    Preconditions.checkArgument(buffer.dataType() == DataType.UTF8, "This method can be called on UTF8 buffers only");
    try {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(bos);

        val numWords = this.length();
        val ub = (Utf8Buffer) buffer;
        // writing length first
        val t = length();
        val ptr = (BytePointer) ub.pointer();

        // now write all strings as bytes
        for (int i = 0; i < ub.length(); i++) {
            dos.writeByte(ptr.get(i));
        }

        val bytes = bos.toByteArray();
        return FlatArray.createBufferVector(builder, bytes);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void testBasicStrings_4a() {
    val arrayX = Nd4j.scalar("alpha");

    val fb = new FlatBufferBuilder();
    val i = arrayX.toFlatArray(fb);
    fb.finish(i);
    val db = fb.dataBuffer();

    val flat = FlatArray.getRootAsFlatArray(db);
    val restored = Nd4j.createFromFlatArray(flat);

    assertEquals("alpha", arrayX.getString(0));

    assertEquals(arrayX, restored);
    assertEquals("alpha", restored.getString(0));
}
 

/**
 * Register the event name - "accuracy", "loss", etc for later use in recording events.
 * @param name Name to register
 * @return Number of bytes written
 */
public long registerEventName(String name) throws IOException {
    Preconditions.checkState(endStaticInfoOffset >= 0, "Cannot write name - have not written end of static info marker");

    FlatBufferBuilder fbb = new FlatBufferBuilder(0);
    long time = System.currentTimeMillis();
    int offset = UIEvent.createUIEvent(fbb, UIEventType.ADD_NAME, UIEventSubtype.NONE, -1, time, 0, 0, (short)-1, 0, 0);
    fbb.finish(offset);

    FlatBufferBuilder fbb2 = new FlatBufferBuilder(0);
    int idx = nameIndexCounter.getAndIncrement();
    nameIndexMap.put(idx, name);
    indexNameMap.put(name, idx);
    int strOffset = fbb2.createString(name);
    int offset2 = UIAddName.createUIAddName(fbb2, idx, strOffset);
    fbb2.finish(offset2);

    long l = append(fbb, fbb2);
    return l;
}
 

/**
 * Convert an {@link INDArray}
 * to an arrow {@link Tensor}
 * @param arr the array to convert
 * @return the equivalent {@link Tensor}
 */
public static Tensor toTensor(INDArray arr) {
    FlatBufferBuilder bufferBuilder = new FlatBufferBuilder(1024);
    long[] strides = getArrowStrides(arr);
    int shapeOffset = createDims(bufferBuilder,arr);
    int stridesOffset = Tensor.createStridesVector(bufferBuilder,strides);

    Tensor.startTensor(bufferBuilder);

    addTypeTypeRelativeToNDArray(bufferBuilder,arr);
    Tensor.addShape(bufferBuilder,shapeOffset);
    Tensor.addStrides(bufferBuilder,stridesOffset);

    Tensor.addData(bufferBuilder,addDataForArr(bufferBuilder,arr));
    int endTensor = Tensor.endTensor(bufferBuilder);
    Tensor.finishTensorBuffer(bufferBuilder,endTensor);
    return Tensor.getRootAsTensor(bufferBuilder.dataBuffer());
}
 

public long writeHistogramEventEqualSpacing(String name, EventSubtype subtype, long time, int iteration, int epoch, double min, double max, INDArray y) throws IOException {
    Preconditions.checkState(y.rank() == 1, "Y array must be rank 1, got Y array with shape %ndShape", y);
    Preconditions.checkState(max > min, "Maximum histogram value must be greater than minimum - got max=%s, min=%s", max, min);

    //TODO add support for plugin, variable and frame/iter
    //TODO: Code duplication for histogram methods...
    Preconditions.checkState(indexNameMap.containsKey(name), "Name \"%s\" not yet registered", name);
    int idx = indexNameMap.get(name);

    FlatBufferBuilder fbb = new FlatBufferBuilder(0);
    int offset = UIEvent.createUIEvent(fbb, UIEventType.HISTOGRAM, subtype.asUIEventSubtype(), idx, time, iteration, epoch, (short)-1, 0, 0);
    fbb.finish(offset);

    FlatBufferBuilder fbb2 = new FlatBufferBuilder(0);
    int yOffset = y.toFlatArray(fbb2);

    INDArray binRangesArr = Nd4j.createFromArray(min, max);
    int binRangesOffset = binRangesArr.toFlatArray(fbb2);

    int offset2 = UIHistogram.createUIHistogram(fbb2, UIHistogramType.EQUAL_SPACING, y.length(), binRangesOffset, yOffset, 0);
    fbb2.finish(offset2);

    return append(fbb, fbb2);
}
 

/**
 * Converting old Arrow Schema to new one based on Arrow version used
 * in Dremio as of 2.1.0
 * @param oldSchema
 * @return
 */
@VisibleForTesting
byte[] convertFromOldSchema(OldSchema oldSchema) {
  FlatBufferBuilder builder = new FlatBufferBuilder();
  int[] fieldOffsets = new int[oldSchema.fieldsLength()];
  for (int i = 0; i < oldSchema.fieldsLength(); i++) {
    fieldOffsets[i] = convertFromOldField(oldSchema.fields(i), builder);
  }
  int fieldsOffset = org.apache.arrow.flatbuf.Schema.createFieldsVector(builder, fieldOffsets);
  int[] metadataOffsets = new int[oldSchema.customMetadataLength()];
  for (int i = 0; i < metadataOffsets.length; i++) {
    int keyOffset = builder.createString(oldSchema.customMetadata(i).key());
    int valueOffset = builder.createString(oldSchema.customMetadata(i).value());
    KeyValue.startKeyValue(builder);
    KeyValue.addKey(builder, keyOffset);
    KeyValue.addValue(builder, valueOffset);
    metadataOffsets[i] = KeyValue.endKeyValue(builder);
  }
  int metadataOffset = org.apache.arrow.flatbuf.Field.createCustomMetadataVector(builder, metadataOffsets);
  org.apache.arrow.flatbuf.Schema.startSchema(builder);
  org.apache.arrow.flatbuf.Schema.addFields(builder, fieldsOffset);
  org.apache.arrow.flatbuf.Schema.addCustomMetadata(builder, metadataOffset);
  builder.finish(org.apache.arrow.flatbuf.Schema.endSchema(builder));
  return builder.sizedByteArray();
}
 

public long writeHistogramEventCustomBins(String name, EventSubtype subtype, long time, int iteration, int epoch, INDArray bins, INDArray y) throws IOException {
    Preconditions.checkState(y.rank() == 1, "Y array must be rank 1, got Y array with shape %ndShape", y);
    Preconditions.checkState(bins.rank() == 2, "Bins array must have shape [2,numBins], got bins array with shape %ndShape", bins);
    Preconditions.checkState(y.length() == bins.size(1), "Bins array must have shape [2,numBins], where numBins must match y.length()=%s, got bins array with shape %ndShape", y.length(), bins);

    //TODO add support for plugin, variable and frame/iter
    //TODO: Code duplication for histogram methods...
    Preconditions.checkState(indexNameMap.containsKey(name), "Name \"%s\" not yet registered", name);
    int idx = indexNameMap.get(name);

    FlatBufferBuilder fbb = new FlatBufferBuilder(0);
    int offset = UIEvent.createUIEvent(fbb, UIEventType.HISTOGRAM, subtype.asUIEventSubtype(), idx, time, iteration, epoch, (short)-1, 0, 0);
    fbb.finish(offset);

    FlatBufferBuilder fbb2 = new FlatBufferBuilder(0);
    int yOffset = y.toFlatArray(fbb2);

    int binRangesOffset = bins.toFlatArray(fbb2);

    int offset2 = UIHistogram.createUIHistogram(fbb2, UIHistogramType.CUSTOM, y.length(), binRangesOffset, yOffset, 0);
    fbb2.finish(offset2);

    return append(fbb, fbb2);
}
 

@Test
public void testByteArrayOrder2() {
    val original = Nd4j.linspace(1, 25, 25, DataType.FLOAT).reshape(5, 5);
    val bufferBuilder = new FlatBufferBuilder(0);

    int array = original.toFlatArray(bufferBuilder);
    bufferBuilder.finish(array);

    val flatArray = FlatArray.getRootAsFlatArray(bufferBuilder.dataBuffer());

    val restored = Nd4j.createFromFlatArray(flatArray);

    assertEquals(original, restored);
}
 

public FragmentWritableBatch(
    final QueryId queryId,
    final int sendMajorFragmentId,
    final int sendMinorFragmentId,
    final int receiveMajorFragmentId,
    ArrowRecordBatch recordBatch,
    final int... receiveMinorFragmentId){
  this.buffers = recordBatch.getBuffers().stream().map(buf -> buf.asNettyBuffer()).collect
    (Collectors.toList()).toArray(new ByteBuf[0]);
  this.recordCount = recordBatch.getLength();
  FlatBufferBuilder fbbuilder = new FlatBufferBuilder();
  fbbuilder.finish(recordBatch.writeTo(fbbuilder));
  ByteBuffer arrowRecordBatch = fbbuilder.dataBuffer();
  final FragmentRecordBatch.Builder builder = FragmentRecordBatch.newBuilder()
      .setArrowRecordBatch(ByteString.copyFrom(arrowRecordBatch))
      .setQueryId(queryId)
      .setReceivingMajorFragmentId(receiveMajorFragmentId)
      .setSendingMajorFragmentId(sendMajorFragmentId)
      .setSendingMinorFragmentId(sendMinorFragmentId);

  for(final int i : receiveMinorFragmentId){
    builder.addReceivingMinorFragmentId(i);
  }

  this.header = builder.build();
}
 

/**
 * This method
 * @param builder
 * @return
 */
public int getFlatConfiguration(FlatBufferBuilder builder) {

    byte prof = profilingMode == OpExecutioner.ProfilingMode.INF_PANIC ? ProfilingMode.INF_PANIC :
                profilingMode == OpExecutioner.ProfilingMode.NAN_PANIC ? ProfilingMode.NAN_PANIC :
                profilingMode == OpExecutioner.ProfilingMode.ANY_PANIC ? ProfilingMode.ANY_PANIC : ProfilingMode.NONE;

    byte exec = executionMode == ExecutionMode.SEQUENTIAL ? org.nd4j.graph.ExecutionMode.SEQUENTIAL :
                executionMode == ExecutionMode.AUTO ? org.nd4j.graph.ExecutionMode.AUTO :
                executionMode == ExecutionMode.STRICT ? org.nd4j.graph.ExecutionMode.STRICT : -1;

    byte outp = outputMode == OutputMode.IMPLICIT ? org.nd4j.graph.OutputMode.IMPLICIT :
                outputMode == OutputMode.EXPLICIT ? org.nd4j.graph.OutputMode.EXPLICIT :
                outputMode == OutputMode.EXPLICIT_AND_IMPLICIT ? org.nd4j.graph.OutputMode.EXPLICIT_AND_IMPLICIT :
                outputMode == OutputMode.VARIABLE_SPACE ? org.nd4j.graph.OutputMode.VARIABLE_SPACE : -1;

    if (exec == -1)
        throw new UnsupportedOperationException("Unknown values were passed into configuration as ExecutionMode: [" + executionMode + "]");

    if (outp == -1)
        throw new UnsupportedOperationException("Unknown values were passed into configuration as OutputMode: [" + outputMode + "]");

    return FlatConfiguration.createFlatConfiguration(builder, -1, prof, exec, outp, gatherTimings, footprintForward, footprintBackward, Direction.FORWARD_ONLY);
}
 

/**
 * This method converts this FunctionProperties instance to FlatBuffers representation
 * @param bufferBuilder
 * @return
 */
public int asFlatProperties(FlatBufferBuilder bufferBuilder) {
    int iname = bufferBuilder.createString(name);
    int ii = FlatProperties.createIVector(bufferBuilder, Ints.toArray(i));
    int il = FlatProperties.createLVector(bufferBuilder, Longs.toArray(l));
    int id = FlatProperties.createDVector(bufferBuilder, Doubles.toArray(d));

    int arrays[] = new int[a.size()];
    int cnt = 0;
    for (val array: a) {
        int off = array.toFlatArray(bufferBuilder);
        arrays[cnt++] = off;
    }

    int ia = FlatProperties.createAVector(bufferBuilder, arrays);

    return FlatProperties.createFlatProperties(bufferBuilder, iname, ii, il, id, ia);
}
 

/**
 * Convert an {@link INDArray}
 * to an arrow {@link Tensor}
 * @param arr the array to convert
 * @return the equivalent {@link Tensor}
 */
public static Tensor toTensor(INDArray arr) {
    FlatBufferBuilder bufferBuilder = new FlatBufferBuilder(1024);
    long[] strides = getArrowStrides(arr);
    int shapeOffset = createDims(bufferBuilder,arr);
    int stridesOffset = Tensor.createStridesVector(bufferBuilder,strides);

    Tensor.startTensor(bufferBuilder);

    addTypeTypeRelativeToNDArray(bufferBuilder,arr);
    Tensor.addShape(bufferBuilder,shapeOffset);
    Tensor.addStrides(bufferBuilder,stridesOffset);

    Tensor.addData(bufferBuilder,addDataForArr(bufferBuilder,arr));
    int endTensor = Tensor.endTensor(bufferBuilder);
    Tensor.finishTensorBuffer(bufferBuilder,endTensor);
    return Tensor.getRootAsTensor(bufferBuilder.dataBuffer());
}
 

@Test
public void testFlatSerde_3() {
    val arrayX = Nd4j.create(new boolean[]{true, false, true, true}, new  long[]{4}, DataType.BOOL);

    val builder = new FlatBufferBuilder(512);
    val flat = arrayX.toFlatArray(builder);
    builder.finish(flat);
    val db = builder.dataBuffer();

    val flatb = FlatArray.getRootAsFlatArray(db);

    val restored = Nd4j.createFromFlatArray(flatb);

    assertEquals(arrayX, restored);
}
 

private boolean ConnectionDenied(Socket socket, String reason) throws IOException {
  FlatBufferBuilder builder = new FlatBufferBuilder();
  int reasonOffset = builder.createString(reason);
  int connectionDeniedId = ConnectionClosed.createConnectionClosed(builder, reasonOffset);
  int id = com.riiablo.net.packet.bnls.BNLS.createBNLS(builder, BNLSData.ConnectionClosed, connectionDeniedId);
  builder.finish(id);

  ByteBuffer data = builder.dataBuffer();
  OutputStream out = socket.getOutputStream();
  WritableByteChannel channel = Channels.newChannel(out);
  channel.write(data);
  return true;
}
 

@Test
public void testFlatSerde_2() {
    val arrayX = Nd4j.create(new long[]{1, 2, 3, 4}, new  long[]{4}, DataType.LONG);

    val builder = new FlatBufferBuilder(512);
    val flat = arrayX.toFlatArray(builder);
    builder.finish(flat);
    val db = builder.dataBuffer();

    val flatb = FlatArray.getRootAsFlatArray(db);

    val restored = Nd4j.createFromFlatArray(flatb);

    assertEquals(arrayX, restored);
}
 

private int[] encodeStrings(FlatBufferBuilder fbb, List<String> list){
    if(list == null || list.isEmpty())
        return null;
    int[] idx = new int[list.size()];
    for( int i=0; i<idx.length; i++ ){
        idx[i] = fbb.createString(list.get(i));
    }
    return idx;
}
 

private void JoinGame(GameSession session, ResponseListener listener) {
  Gdx.app.debug(TAG, "Joining game " + session);
  FlatBufferBuilder builder = new FlatBufferBuilder();
  int gameNameOffset = builder.createString(session.name);
  JoinGame.startJoinGame(builder);
  JoinGame.addGameName(builder, gameNameOffset);
  int joinGameOffset = JoinGame.endJoinGame(builder);
  int id = MCP.createMCP(builder, MCPData.JoinGame, joinGameOffset);
  builder.finish(id);
  ByteBuffer data = builder.dataBuffer();
  connection.sendRequest(data, listener);
}
 

@Test
public void testByteArrayOrder2() {
    val original = Nd4j.linspace(1, 25, 25).reshape(5, 5);
    val bufferBuilder = new FlatBufferBuilder(0);

    int array = original.toFlatArray(bufferBuilder);
    bufferBuilder.finish(array);

    val flatArray = FlatArray.getRootAsFlatArray(bufferBuilder.dataBuffer());

    val restored = Nd4j.createFromFlatArray(flatArray);

    assertEquals(original, restored);
}
 

@Test
public void testByteArrayOrder3() {
    val original = Nd4j.linspace(1, 25, 25).reshape('f', 5, 5);
    val bufferBuilder = new FlatBufferBuilder(0);

    int array = original.toFlatArray(bufferBuilder);
    bufferBuilder.finish(array);

    val flatArray = FlatArray.getRootAsFlatArray(bufferBuilder.dataBuffer());

    val restored = Nd4j.createFromFlatArray(flatArray);

    assertEquals(original, restored);
}
 

@Test
public void testFlatSerde_1() {
    val arrayX = Nd4j.create(new int[]{1, 2, 3, 4}, new  long[]{4}, DataType.INT);

    val builder = new FlatBufferBuilder(512);
    val flat = arrayX.toFlatArray(builder);
    builder.finish(flat);
    val db = builder.dataBuffer();

    val flatb = FlatArray.getRootAsFlatArray(db);

    val restored = Nd4j.createFromFlatArray(flatb);

    assertEquals(arrayX, restored);
}
 

/**
 * This method converts multiple FunctionProperties to FlatBuffers representation
 *
 * @param bufferBuilder
 * @param properties
 * @return
 */
public static int asFlatProperties(FlatBufferBuilder bufferBuilder, Collection<FunctionProperties> properties) {
    int props[] = new int[properties.size()];

    int cnt = 0;
    for (val p: properties)
        props[cnt++] = p.asFlatProperties(bufferBuilder);

    return FlatNode.createPropertiesVector(bufferBuilder, props);
}
 

private void CreateGame(GameSession session, ResponseListener listener) {
  Gdx.app.debug(TAG, "Creating game " + session);
  FlatBufferBuilder builder = new FlatBufferBuilder();
  int gameNameOffset = builder.createString(session.name);
  int descriptionOffset = builder.createString(session.desc);
  CreateGame.startCreateGame(builder);
  CreateGame.addGameName(builder, gameNameOffset);
  CreateGame.addDescription(builder, descriptionOffset);
  int createGameOffset = CreateGame.endCreateGame(builder);
  int id = MCP.createMCP(builder, MCPData.CreateGame, createGameOffset);
  builder.finish(id);
  ByteBuffer data = builder.dataBuffer();
  connection.sendRequest(data, listener);
}
 

@SuppressWarnings("unchecked")
public int serialize(FlatBufferBuilder builder, int entityId) {
  dataType.clear();
  data.clear();
  components.clear();

  int type = mClass.get(entityId).type.ordinal();

  int flags = mFlags.get(entityId).flags;
  if ((flags & EntityFlags.deleted) == EntityFlags.deleted) {
    int dataTypeOffset = EntitySync.createComponentTypeVector(builder, ArrayUtils.EMPTY_BYTE_ARRAY);
    int dataOffset = EntitySync.createComponentVector(builder, ArrayUtils.EMPTY_INT_ARRAY);
    return EntitySync.createEntitySync(builder, entityId, type, flags, dataTypeOffset, dataOffset);
  }

  componentManager.getComponentsFor(entityId, components);
  for (Component c : components) {
    FlatBuffersSerializer serializer = serializers.get(c.getClass());
    if (serializer == null) continue;
    dataType.add(serializer.getDataType());
    data.add(serializer.putData(builder, c));
  }

  assert dataType.size == data.size;

  final int dataTypeSize = dataType.size;
  final byte[] dataType = this.dataType.items;
  EntitySync.startComponentTypeVector(builder, dataTypeSize);
  for (int i = 0; i < dataTypeSize; i++) builder.addByte(dataType[i]);
  int dataTypeOffset = builder.endVector();

  final int dataSize = data.size;
  final int[] data = this.data.items;
  EntitySync.startComponentVector(builder, dataSize);
  for (int i = 0; i < dataSize; i++) builder.addOffset(data[i]);
  int dataOffset = builder.endVector();

  return EntitySync.createEntitySync(builder, entityId, type, flags, dataTypeOffset, dataOffset);
}
 

/**
 * Write marker for final static data
 * @return
 * @throws IOException
 */
public long writeFinishStaticMarker() throws IOException {
    Preconditions.checkState(endStaticInfoOffset < 0, "Wrote final static already information already");
    Pair<Integer, FlatBufferBuilder> encoded = encodeStaticHeader(UIInfoType.START_EVENTS);
    long out = append(encoded.getSecond(), null);
    endStaticInfoOffset = file.length();
    return out;
}
 

private void Ping(Packet packet) {
  Ping ping = (Ping) packet.data.data(new Ping());
  FlatBufferBuilder builder = new FlatBufferBuilder(0);
  int dataOffset = Ping.createPing(builder, ping.tickCount(), ping.sendTime(), TimeUtils.millis() - packet.time, false);
  int root = com.riiablo.net.packet.d2gs.D2GS.createD2GS(builder, D2GSData.Ping, dataOffset);
  com.riiablo.net.packet.d2gs.D2GS.finishSizePrefixedD2GSBuffer(builder, root);
  Packet response = Packet.obtain(1 << packet.id, builder.dataBuffer());
  outPackets.offer(response);
}
 

@Override
public void cursorToGround() {
  FlatBufferBuilder builder = obtainBuilder();
  CursorToGround.startCursorToGround(builder);
  int dataOffset = CursorToGround.endCursorToGround(builder);
  wrapAndSend(builder, D2GSData.CursorToGround, dataOffset);
}
 

@Override
public void groundToCursor(int entityId) {
  int serverId = mNetworked.get(entityId).serverId;
  FlatBufferBuilder builder = obtainBuilder();
  int dataOffset = GroundToCursor.createGroundToCursor(builder, serverId);
  wrapAndSend(builder, D2GSData.GroundToCursor, dataOffset);
}
 

@Override
public int toFlatArray(FlatBufferBuilder builder) {
    if(isView()){
        return dup(this.ordering()).toFlatArray(builder);
    }
    int shape = FlatArray.createShapeVector(builder, this.shapeInfoDataBuffer().asLong());
    int buffer = this.isEmpty() ? 0 : this.dataType() == DataType.UTF8 ? stringBuffer(builder, this.data()) : FlatArray.createBufferVector(builder, this.data().asBytes());
    val type = this.isEmpty() ? FlatBuffersMapper.getDataTypeAsByte(this.dataType()) : FlatBuffersMapper.getDataTypeAsByte(this.data().dataType());
    int array = FlatArray.createFlatArray(builder, shape, buffer, type, ByteOrder.BE);

    return array;
}
 

@Override
protected void processSystem() {
  FlatBufferBuilder builder = new FlatBufferBuilder(0);
  int dataOffset = Ping.createPing(builder, tick++, TimeUtils.millis(), 0, false);
  int root = D2GS.createD2GS(builder, D2GSData.Ping, dataOffset);
  D2GS.finishSizePrefixedD2GSBuffer(builder, root);

  try {
    OutputStream out = socket.getOutputStream();
    WritableByteChannel channelOut = Channels.newChannel(out);
    channelOut.write(builder.dataBuffer());
  } catch (Throwable t) {
    Gdx.app.error(TAG, t.getMessage(), t);
  }
}
 

private boolean ConnectionAccepted(Socket socket) throws IOException {
  Gdx.app.debug(TAG, "Connection accepted!");
  FlatBufferBuilder builder = new FlatBufferBuilder();
  ConnectionAccepted.startConnectionAccepted(builder);
  int connectionAcceptedId = ConnectionAccepted.endConnectionAccepted(builder);
  int id = com.riiablo.net.packet.bnls.BNLS.createBNLS(builder, BNLSData.ConnectionAccepted, connectionAcceptedId);
  builder.finish(id);

  ByteBuffer data = builder.dataBuffer();
  OutputStream out = socket.getOutputStream();
  WritableByteChannel channel = Channels.newChannel(out);
  channel.write(data);
  return false;
}