Java Code Examples for com.google.flatbuffers.FlatBufferBuilder#dataBuffer()

public FragmentWritableBatch(
    final QueryId queryId,
    final int sendMajorFragmentId,
    final int sendMinorFragmentId,
    final int receiveMajorFragmentId,
    ArrowRecordBatch recordBatch,
    final int... receiveMinorFragmentId){
  this.buffers = recordBatch.getBuffers().stream().map(buf -> buf.asNettyBuffer()).collect
    (Collectors.toList()).toArray(new ByteBuf[0]);
  this.recordCount = recordBatch.getLength();
  FlatBufferBuilder fbbuilder = new FlatBufferBuilder();
  fbbuilder.finish(recordBatch.writeTo(fbbuilder));
  ByteBuffer arrowRecordBatch = fbbuilder.dataBuffer();
  final FragmentRecordBatch.Builder builder = FragmentRecordBatch.newBuilder()
      .setArrowRecordBatch(ByteString.copyFrom(arrowRecordBatch))
      .setQueryId(queryId)
      .setReceivingMajorFragmentId(receiveMajorFragmentId)
      .setSendingMajorFragmentId(sendMajorFragmentId)
      .setSendingMinorFragmentId(sendMinorFragmentId);

  for(final int i : receiveMinorFragmentId){
    builder.addReceivingMinorFragmentId(i);
  }

  this.header = builder.build();
}
 

private boolean QueryRealms(Socket socket) throws IOException {
  FlatBufferBuilder builder = new FlatBufferBuilder();

  int[] realms = new int[REALMS.length];
  for (int i = 0; i < REALMS.length; i++) {
    realms[i] = Realm.createRealm(builder, builder.createString(REALMS[i][0]), builder.createString(REALMS[i][1]));
  }
  int realmsVec = QueryRealms.createRealmsVector(builder, realms);

  QueryRealms.startQueryRealms(builder);
  QueryRealms.addRealms(builder, realmsVec);
  int realmId = QueryRealms.endQueryRealms(builder);

  int id = com.riiablo.net.packet.bnls.BNLS.createBNLS(builder, BNLSData.QueryRealms, realmId);

  builder.finish(id);

  ByteBuffer data = builder.dataBuffer();

  OutputStream out = socket.getOutputStream();
  WritableByteChannel channel = Channels.newChannel(out);
  channel.write(data);
  Gdx.app.log(TAG, "returning realms list...");
  return false;
}
 

private long append(FlatBufferBuilder h, FlatBufferBuilder c) throws IOException {
    ByteBuffer bb1 = h.dataBuffer();
    ByteBuffer bb2 = (c == null ? null : c.dataBuffer());

    try (RandomAccessFile f = new RandomAccessFile(file, "rw"); FileChannel fc = f.getChannel(); FileLock lock = fc.lock()) {
        //TODO can we make this more efficient - use a single byte buffer?

        //Seek to end for append
        f.seek(f.length());
        long startPos = f.getFilePointer();

        //Write header - length of SystemInfo header, length of content header
        ByteBuffer header = ByteBuffer.allocate(8); //8 bytes = 2x 4 byte integers
        int l1 = bb1.remaining();
        int l2 = bb2 == null ? 0 : bb2.remaining();
        header.putInt(l1);
        header.putInt(l2);
        header.flip();

        //System.out.println("Lengths - header, content: " + l1 + ", " + l2);

        int w1 = fc.write(header);
        int w2 = fc.write(bb1);
        int w3 = bb2 == null ? 0 : fc.write(bb2);
        long total = w1 + w2 + w3;
        //System.out.println("Wrote " + total + " bytes starting at position " + startPos);
        //System.out.println("Post writing file length: " + file.length());
        return total;
    }
}
 

@Test
public void testVectorEncoding_2() {
    val scalar = Nd4j.createFromArray(new double[]{1, 2, 3, 4, 5});

    FlatBufferBuilder bufferBuilder = new FlatBufferBuilder(0);
    val fb = scalar.toFlatArray(bufferBuilder);
    bufferBuilder.finish(fb);
    val db = bufferBuilder.dataBuffer();

    val flat = FlatArray.getRootAsFlatArray(db);

    val restored = Nd4j.createFromFlatArray(flat);

    assertEquals(scalar, restored);
}
 

@Test
public void testVectorEncoding_1() {
    val scalar = Nd4j.createFromArray(new float[]{1, 2, 3, 4, 5});

    FlatBufferBuilder bufferBuilder = new FlatBufferBuilder(0);
    val fb = scalar.toFlatArray(bufferBuilder);
    bufferBuilder.finish(fb);
    val db = bufferBuilder.dataBuffer();

    val flat = FlatArray.getRootAsFlatArray(db);

    val restored = Nd4j.createFromFlatArray(flat);

    assertEquals(scalar, restored);
}
 

private void JoinGame(GameSession session, ResponseListener listener) {
  Gdx.app.debug(TAG, "Joining game " + session);
  FlatBufferBuilder builder = new FlatBufferBuilder();
  int gameNameOffset = builder.createString(session.name);
  JoinGame.startJoinGame(builder);
  JoinGame.addGameName(builder, gameNameOffset);
  int joinGameOffset = JoinGame.endJoinGame(builder);
  int id = MCP.createMCP(builder, MCPData.JoinGame, joinGameOffset);
  builder.finish(id);
  ByteBuffer data = builder.dataBuffer();
  connection.sendRequest(data, listener);
}
 

private void CreateGame(GameSession session, ResponseListener listener) {
  Gdx.app.debug(TAG, "Creating game " + session);
  FlatBufferBuilder builder = new FlatBufferBuilder();
  int gameNameOffset = builder.createString(session.name);
  int descriptionOffset = builder.createString(session.desc);
  CreateGame.startCreateGame(builder);
  CreateGame.addGameName(builder, gameNameOffset);
  CreateGame.addDescription(builder, descriptionOffset);
  int createGameOffset = CreateGame.endCreateGame(builder);
  int id = MCP.createMCP(builder, MCPData.CreateGame, createGameOffset);
  builder.finish(id);
  ByteBuffer data = builder.dataBuffer();
  connection.sendRequest(data, listener);
}
 

private void ListGames(ResponseListener listener) {
  Gdx.app.debug(TAG, "Requesting games list");
  FlatBufferBuilder builder = new FlatBufferBuilder();
  ListGames.startListGames(builder);
  int listGamesOffset = ListGames.endListGames(builder);
  int id = MCP.createMCP(builder, MCPData.ListGames, listGamesOffset);
  builder.finish(id);
  ByteBuffer data = builder.dataBuffer();
  connection.sendRequest(data, listener);
}
 

private boolean ConnectionAccepted(Socket socket) throws IOException {
  Gdx.app.debug(TAG, "Connection accepted!");
  FlatBufferBuilder builder = new FlatBufferBuilder();
  ConnectionAccepted.startConnectionAccepted(builder);
  int connectionAcceptedId = ConnectionAccepted.endConnectionAccepted(builder);
  int id = com.riiablo.net.packet.bnls.BNLS.createBNLS(builder, BNLSData.ConnectionAccepted, connectionAcceptedId);
  builder.finish(id);

  ByteBuffer data = builder.dataBuffer();
  OutputStream out = socket.getOutputStream();
  WritableByteChannel channel = Channels.newChannel(out);
  channel.write(data);
  return false;
}
 

private boolean ConnectionDenied(Socket socket, String reason) throws IOException {
  FlatBufferBuilder builder = new FlatBufferBuilder();
  int reasonOffset = builder.createString(reason);
  int connectionDeniedId = ConnectionClosed.createConnectionClosed(builder, reasonOffset);
  int id = com.riiablo.net.packet.bnls.BNLS.createBNLS(builder, BNLSData.ConnectionClosed, connectionDeniedId);
  builder.finish(id);

  ByteBuffer data = builder.dataBuffer();
  OutputStream out = socket.getOutputStream();
  WritableByteChannel channel = Channels.newChannel(out);
  channel.write(data);
  return true;
}
 

private boolean StartInstance(Socket socket, com.riiablo.net.packet.msi.MSI packet) throws IOException {
  Gdx.app.debug(TAG, "Starting instance...");

  try {
    File outFile = new File("D2GS.tmp");
    ProcessBuilder processBuilder = new ProcessBuilder("java", "-jar", "server/d2gs/build/libs/d2gs-1.0.jar");
    processBuilder.redirectOutput(ProcessBuilder.Redirect.to(outFile));
    processBuilder.redirectError(ProcessBuilder.Redirect.to(outFile));
    Process process = processBuilder.start();
    instances.add(process);
  } catch (Throwable t) {
    Gdx.app.error(TAG, t.getMessage(), t);
  }

  int ip     = 2130706433; // 127.0.0.1
  short port = 6114;

  FlatBufferBuilder builder = new FlatBufferBuilder();
  StartInstance.startStartInstance(builder);
  StartInstance.addResult(builder, Result.SUCCESS);
  StartInstance.addIp(builder, ip);
  StartInstance.addPort(builder, port);
  int startInstanceOffset = StartInstance.endStartInstance(builder);
  int id = com.riiablo.net.packet.msi.MSI.createMSI(builder, MSIData.StartInstance, startInstanceOffset);
  builder.finish(id);

  ByteBuffer data = builder.dataBuffer();
  OutputStream out = socket.getOutputStream();
  WritableByteChannel channel = Channels.newChannel(out);
  channel.write(data);
  Gdx.app.debug(TAG, "Returning instance at " + InetAddress.getByAddress(ByteBuffer.allocate(4).putInt(ip).array()));
  return true;
}
 

private void StartInstance(CreateGame createGame, ResponseListener listener) {
  Gdx.app.debug(TAG, "Requesting game instance for " + createGame);
  FlatBufferBuilder builder = new FlatBufferBuilder();
  StartInstance.startStartInstance(builder);
  int startInstanceOffset = StartInstance.endStartInstance(builder);
  int id = MSI.createMSI(builder, MSIData.StartInstance, startInstanceOffset);
  builder.finish(id);
  ByteBuffer data = builder.dataBuffer();

  Socket socket = null;
  try {
    socket = Gdx.net.newClientSocket(Net.Protocol.TCP, "localhost", com.riiablo.server.mcp.MSI.PORT, null);

    OutputStream out = socket.getOutputStream();
    WritableByteChannel channelOut = Channels.newChannel(out);
    channelOut.write(data);

    buffer.clear();
    buffer.mark();
    InputStream in = socket.getInputStream();
    ReadableByteChannel channelIn = Channels.newChannel(in);
    channelIn.read(buffer);
    buffer.limit(buffer.position());
    buffer.reset();

    MSI packet = MSI.getRootAsMSI(buffer);
    Gdx.app.log(TAG, "packet type " + MCPData.name(packet.dataType()));
    listener.handleResponse(packet);
  } catch (Throwable t) {
    listener.failed(t);
  } finally {
    if (socket != null) socket.dispose();
  }
}
 

private boolean ConnectionAccepted(Socket socket) throws IOException {
  Gdx.app.debug(TAG, "Connection accepted!");
  FlatBufferBuilder builder = new FlatBufferBuilder();
  ConnectionAccepted.startConnectionAccepted(builder);
  int connectionAcceptedId = ConnectionAccepted.endConnectionAccepted(builder);
  int id = com.riiablo.net.packet.mcp.MCP.createMCP(builder, MCPData.ConnectionAccepted, connectionAcceptedId);
  builder.finish(id);

  ByteBuffer data = builder.dataBuffer();
  OutputStream out = socket.getOutputStream();
  WritableByteChannel channel = Channels.newChannel(out);
  channel.write(data);
  return false;
}
 

private boolean ConnectionDenied(Socket socket, String reason) throws IOException {
  FlatBufferBuilder builder = new FlatBufferBuilder();
  int reasonOffset = builder.createString(reason);
  int connectionDeniedId = ConnectionClosed.createConnectionClosed(builder, reasonOffset);
  int id = com.riiablo.net.packet.mcp.MCP.createMCP(builder, MCPData.ConnectionClosed, connectionDeniedId);
  builder.finish(id);

  ByteBuffer data = builder.dataBuffer();
  OutputStream out = socket.getOutputStream();
  WritableByteChannel channel = Channels.newChannel(out);
  channel.write(data);
  return true;
}
 

@Test
public void testScalarEncoding() {
    val scalar = Nd4j.trueScalar(2.0f);

    FlatBufferBuilder bufferBuilder = new FlatBufferBuilder(0);
    val fb = scalar.toFlatArray(bufferBuilder);
    bufferBuilder.finish(fb);
    val db = bufferBuilder.dataBuffer();

    val flat = FlatArray.getRootAsFlatArray(db);


    val restored = Nd4j.createFromFlatArray(flat);

    assertEquals(scalar, restored);
}
 

@Test
public void testVectorEncoding() {
    val scalar = Nd4j.trueVector(new float[]{1, 2, 3, 4, 5});

    FlatBufferBuilder bufferBuilder = new FlatBufferBuilder(0);
    val fb = scalar.toFlatArray(bufferBuilder);
    bufferBuilder.finish(fb);
    val db = bufferBuilder.dataBuffer();

    val flat = FlatArray.getRootAsFlatArray(db);


    val restored = Nd4j.createFromFlatArray(flat);

    assertEquals(scalar, restored);
}
 

@Test
public void testScalarEncoding() {
    val scalar = Nd4j.scalar(2.0f);

    FlatBufferBuilder bufferBuilder = new FlatBufferBuilder(0);
    val fb = scalar.toFlatArray(bufferBuilder);
    bufferBuilder.finish(fb);
    val db = bufferBuilder.dataBuffer();

    val flat = FlatArray.getRootAsFlatArray(db);


    val restored = Nd4j.createFromFlatArray(flat);

    assertEquals(scalar, restored);
}
 

public static ByteBuffer serializeRecordToFlatBuffer(FlatBufferBuilder builder, Record<byte[]> record) {
    checkNotNull(record, "record-context can't be null");
    Optional<EncryptionContext> encryptionCtx = (record instanceof RecordWithEncryptionContext)
            ? ((RecordWithEncryptionContext<byte[]>) record).getEncryptionCtx()
            : Optional.empty();
    Map<String, String> properties = record.getProperties();

    int encryptionCtxOffset = -1;
    int propertiesOffset = -1;

    if (properties != null && !properties.isEmpty()) {
        int[] propertiesOffsetArray = new int[properties.size()];
        int i = 0;
        for (Entry<String, String> property : properties.entrySet()) {
            propertiesOffsetArray[i++] = KeyValue.createKeyValue(builder, builder.createString(property.getKey()),
                    builder.createString(property.getValue()));
        }
        propertiesOffset = Message.createPropertiesVector(builder, propertiesOffsetArray);
    }

    if (encryptionCtx.isPresent()) {
        encryptionCtxOffset = createEncryptionCtxOffset(builder, encryptionCtx);
    }

    int payloadOffset = Message.createPayloadVector(builder, record.getValue());
    Message.startMessage(builder);
    Message.addPayload(builder, payloadOffset);
    if (encryptionCtxOffset != -1) {
        Message.addEncryptionCtx(builder, encryptionCtxOffset);
    }
    if (propertiesOffset != -1) {
        Message.addProperties(builder, propertiesOffset);
    }
    int endMessage = Message.endMessage(builder);
    builder.finish(endMessage);
    ByteBuffer bb = builder.dataBuffer();

    // to avoid copying of data, use same byte[] wrapped by ByteBuffer. But, ByteBuffer.array() returns entire array
    // so, it requires to read from offset:
    // builder.sizedByteArray()=>copies buffer: sizedByteArray(space, bb.capacity() - space)
    int space = bb.capacity() - builder.offset();
    return ByteBuffer.wrap(bb.array(), space, bb.capacity() - space);
}
 

@Test
public void testPutMultipleBinary() throws Exception {

    FlatBufferBuilder builder = new FlatBufferBuilder(1);

    int[] metric = new int[2];
    Map<String, String> t = new HashMap<>();
    t.put("tag1", "value1");
    t.put("tag2", "value2");
    metric[0] = createMetric(builder, "sys.cpu.user", TEST_TIME, 1.0D, t);
    t = new HashMap<>();
    t.put("tag3", "value3");
    t.put("tag4", "value4");
    metric[1] = createMetric(builder, "sys.cpu.idle", TEST_TIME + 1, 1.0D, t);

    int metricVector = timely.api.flatbuffer.Metrics.createMetricsVector(builder, metric);

    timely.api.flatbuffer.Metrics.startMetrics(builder);
    timely.api.flatbuffer.Metrics.addMetrics(builder, metricVector);
    int metrics = timely.api.flatbuffer.Metrics.endMetrics(builder);
    timely.api.flatbuffer.Metrics.finishMetricsBuffer(builder, metrics);

    ByteBuffer binary = builder.dataBuffer();
    byte[] data = new byte[binary.remaining()];
    binary.get(data, 0, binary.remaining());
    LOG.debug("Sending {} bytes", data.length);

    final TestServer m = new TestServer(conf);
    m.run();
    try (Socket sock = new Socket("127.0.0.1", 54321);) {
        sock.getOutputStream().write(data);
        sock.getOutputStream().flush();
        while (2 != m.getTcpRequests().getCount()) {
            LOG.debug("Thread sleeping");
            Thread.sleep(5);
        }
        Assert.assertEquals(2, m.getTcpRequests().getResponses().size());
        Assert.assertEquals(MetricRequest.class, m.getTcpRequests().getResponses().get(0).getClass());
        // @formatter:off
        MetricRequest actual = (MetricRequest) m.getTcpRequests().getResponses().get(0);
        MetricRequest expected = new MetricRequest(
                Metric.newBuilder()
                        .name("sys.cpu.user")
                        .value(TEST_TIME, 1.0D)
                        .tag(new Tag("tag1", "value1"))
                        .tag(new Tag("tag2", "value2"))
                        .build()
        );
        Assert.assertEquals(expected, actual);

        Assert.assertEquals(MetricRequest.class, m.getTcpRequests().getResponses().get(1).getClass());
        actual = (MetricRequest) m.getTcpRequests().getResponses().get(1);
        expected = new MetricRequest(
                Metric.newBuilder()
                        .name("sys.cpu.idle")
                        .value(TEST_TIME + 1, 1.0D)
                        .tag(new Tag("tag3", "value3"))
                        .tag(new Tag("tag4", "value4"))
                        .build()
        );
        // @formatter:on
        Assert.assertEquals(expected, actual);

    } finally {
        m.shutdown();
    }
}
 

@Test
public void testPutMultipleBinary() throws Exception {

    FlatBufferBuilder builder = new FlatBufferBuilder(1);

    int[] metric = new int[2];
    Map<String, String> t = new HashMap<>();
    t.put("tag1", "value1");
    t.put("tag2", "value2");
    metric[0] = createMetric(builder, "sys.cpu.user", TEST_TIME, 1.0D, t);
    t = new HashMap<>();
    t.put("tag3", "value3");
    t.put("tag4", "value4");
    metric[1] = createMetric(builder, "sys.cpu.idle", TEST_TIME + 1, 1.0D, t);

    int metricVector = timely.api.flatbuffer.Metrics.createMetricsVector(builder, metric);

    timely.api.flatbuffer.Metrics.startMetrics(builder);
    timely.api.flatbuffer.Metrics.addMetrics(builder, metricVector);
    int metrics = timely.api.flatbuffer.Metrics.endMetrics(builder);
    timely.api.flatbuffer.Metrics.finishMetricsBuffer(builder, metrics);

    ByteBuffer binary = builder.dataBuffer();
    byte[] data = new byte[binary.remaining()];
    binary.get(data, 0, binary.remaining());
    LOG.debug("Sending {} bytes", data.length);

    final TestServer m = new TestServer(conf);
    m.run();
    InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
    DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
    try (DatagramSocket sock = new DatagramSocket()) {
        packet.setData(data);
        sock.send(packet);
        while (2 != m.getUdpRequests().getCount()) {
            Thread.sleep(5);
        }
        Assert.assertEquals(2, m.getUdpRequests().getResponses().size());
        Assert.assertEquals(MetricRequest.class, m.getUdpRequests().getResponses().get(0).getClass());
        // @formatter:off
        MetricRequest actual = (MetricRequest) m.getUdpRequests().getResponses().get(0);
        MetricRequest expected = new MetricRequest(
                Metric.newBuilder()
                        .name("sys.cpu.user")
                        .value(TEST_TIME, 1.0D)
                        .tag(new Tag("tag1", "value1"))
                        .tag(new Tag("tag2", "value2"))
                        .build()
        );
        Assert.assertEquals(expected, actual);

        Assert.assertEquals(MetricRequest.class, m.getUdpRequests().getResponses().get(1).getClass());
        actual = (MetricRequest) m.getUdpRequests().getResponses().get(1);
        expected = new MetricRequest(
                Metric.newBuilder()
                        .name("sys.cpu.idle")
                        .value(TEST_TIME + 1, 1.0D)
                        .tag(new Tag("tag3", "value3"))
                        .tag(new Tag("tag4", "value4"))
                        .build()
        );
        Assert.assertEquals(expected, actual);
        // @formatter:on

    } finally {
        m.shutdown();
    }
}