Java Code Examples for org.apache.thrift.TSerializer#serialize()

@Test
public void testDeserializeColumn() throws Exception {
  Mutation mutation =
      new Mutation(
          MutationType.DELETE,
          TIMESTAMP,
          SOURCE_ID,
          DATA_SOURCE,
          BINLOG_HEADER,
          TABLE,
          getEntity());

  TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
  TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());

  byte[] serialized = serializer.serialize(mutation);

  Mutation deserialized = new Mutation();
  deserializer.deserialize(deserialized, serialized);

  assertEquals(mutation, deserialized);
}
 

public static String pack(GeoXPShape shape) throws WarpScriptException {
  long[] cells = GeoXPLib.getCells(shape);
  
  GTSEncoder encoder = new GTSEncoder();
  
  try {
    for (long cell: cells) {
      encoder.addValue(cell, GeoTimeSerie.NO_LOCATION, GeoTimeSerie.NO_ELEVATION, true);
    }      
  } catch (IOException ioe) {
    throw new WarpScriptException(ioe);
  }
  
  GTSWrapper wrapper = GTSWrapperHelper.fromGTSEncoderToGTSWrapper(encoder, true);
  
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  
  try {
    byte[] serialized = serializer.serialize(wrapper);
    
    return new String(OrderPreservingBase64.encode(serialized, 0, serialized.length), StandardCharsets.US_ASCII);
  } catch (TException te) {
    throw new WarpScriptException(te);
  }
}
 

public static final String serializeLoggingEvent(KeyStore keystore, LoggingEvent event) {
  if (null == event) {
    return null;
  }
  
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  
  byte[] serialized = null;
  
  try {
    serialized = serializer.serialize(event);
  } catch (TException te) {
    return null;
  }
  
  if (!checkedAESKey) {
    checkedAESKey = true;
    loggingAESKey = keystore.getKey(KeyStore.AES_LOGGING);      
  }
  if (null != loggingAESKey) {
    serialized = CryptoUtils.wrap(loggingAESKey, serialized);
  }
  
  return new String(OrderPreservingBase64.encode(serialized), StandardCharsets.US_ASCII);
}
 

public static byte[] serialize(ThriftProtocol protocol, TBase<?, ?> source)
    throws ThriftException {
  TSerializer serializer = new TSerializer(getProtocolFactory(protocol));
  try {
    return serializer.serialize(source);
  } catch (TException e) {
    throw new ThriftException(e);
  }
}
 

public static byte[] toBytes(TBase event) {
	TSerializer tSerializer = tSerializerLocal.get();
	byte[] data;
	try {
		data = tSerializer.serialize(event);
	} catch (TException e) {
		throw new RuntimeException("fail to serialize thrift object of type " + event.getClass());
	}
	return data;
}
 

public static String thriftToDebugJson(TBase<?, ?> thriftObject) {
  TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory());
  try {
    return new String(serializer.serialize(thriftObject));
  } catch (TException e) {
    LOGGER.error(
        e,
        String.format(
            "Failed trying to serialize type [%s] to debug JSON.",
            thriftObject.getClass().getName()));
    return "FAILED_TO_DESERIALIZE";
  }
}
 

private Message buildMessage(long timestamp, int timestampTwo, long timestampThree) throws Exception {
    UnitTestMessage thriftMessage = new UnitTestMessage(timestamp, "notimportant", timestampTwo, timestampThree);
    TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
    byte[] data = serializer.serialize(thriftMessage);

    return new Message("test", 0, 0, null, data, timestamp, null);
}
 

public static byte[] thriftSerialize(TBase t) {
    try {
        TSerializer ser = getSer();
        return ser.serialize(t);
    } catch (TException e) {
        LOG.error("Failed to serialize to thrift: ", e);
        throw new RuntimeException(e);
    }
}
 

@SuppressWarnings("rawtypes")
public static byte[] serialize(TBase baseObject) throws IOException {
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory(maxMessageSize, maxMessageSize));
  try {
    return serializer.serialize(baseObject);
  } catch (TException e) {
    throw new IOException("Error serializing thrift object "
        + baseObject, e);
  }
}
 

@Override
public @Nullable ByteBuf encodeBody(@NotNull Object body) {
    try {
        TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
        byte[] payloadBytes = serializer.serialize((TBase<?, ?>) body);
        return Unpooled.wrappedBuffer(payloadBytes);
    } catch (TException e) {
        logger.error("Failed to encode {} body", body.getClass().getName(), e);
    }
    return null;
}
 

public static synchronized boolean offer(WebCallRequest request) {    
  try {
    //
    // Initialize service if not done yet
    //
    
    if (!initialized) {
      initialize();
    }
  
    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
    
    byte[] value = serializer.serialize(request);
    
    //
    // Wrap data if AES key is defined
    //
    
    if (null != aesKey) {
      value = CryptoUtils.wrap(aesKey, value);               
    }
    
    //
    // Compute MAC if the SipHash key is defined
    //
    
    if (null != siphashKey) {
      value = CryptoUtils.addMAC(siphashKey, value);
    }

    KeyedMessage<byte[], byte[]> message = new KeyedMessage<byte[], byte[]>(topic, value);
    
    producer.send(message);
    
    return true;
  } catch (Exception e) {
    return false;
  }
}
 

@Test
public void testThriftParquetReadWriteRoundTrip() throws Exception {
    Map<String, String> classPerTopic = new HashMap<String, String>();
    classPerTopic.put("test-pb-topic", UnitTestMessage.class.getName());
    Mockito.when(config.getThriftMessageClassPerTopic()).thenReturn(classPerTopic);
    Mockito.when(config.getFileReaderWriterFactory())
            .thenReturn(ThriftParquetFileReaderWriterFactory.class.getName());
    Mockito.when(config.getThriftProtocolClass())
    .thenReturn(TCompactProtocol.class.getName());
    Mockito.when(ParquetUtil.getParquetBlockSize(config))
            .thenReturn(ParquetWriter.DEFAULT_BLOCK_SIZE);
    Mockito.when(ParquetUtil.getParquetPageSize(config))
            .thenReturn(ParquetWriter.DEFAULT_PAGE_SIZE);
    Mockito.when(ParquetUtil.getParquetEnableDictionary(config))
            .thenReturn(ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED);
    Mockito.when(ParquetUtil.getParquetValidation(config))
            .thenReturn(ParquetWriter.DEFAULT_IS_VALIDATING_ENABLED);


    LogFilePath tempLogFilePath = new LogFilePath(Files.createTempDir().toString(), "test-pb-topic",
            new String[] { "part-1" }, 0, 1, 23232, ".log");

    FileWriter fileWriter = ReflectionUtil.createFileWriter(config.getFileReaderWriterFactory(), tempLogFilePath,
            null, config);

    UnitTestMessage msg1 = new UnitTestMessage().setRequiredField("abc").setTimestamp(1467176315L);
    UnitTestMessage msg2 = new UnitTestMessage().setRequiredField("XYZ").setTimestamp(1467176344L);

    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
    KeyValue kv1 = new KeyValue(23232, serializer.serialize(msg1));
    KeyValue kv2 = new KeyValue(23233, serializer.serialize(msg2));
    fileWriter.write(kv1);
    fileWriter.write(kv2);
    fileWriter.close();

    FileReader fileReader = ReflectionUtil.createFileReader(config.getFileReaderWriterFactory(), tempLogFilePath,
            null, config);
    TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());
    
    KeyValue kvout = fileReader.next();
    assertEquals(kv1.getOffset(), kvout.getOffset());
    assertArrayEquals(kv1.getValue(), kvout.getValue());
    UnitTestMessage actual = new UnitTestMessage();
    deserializer.deserialize(actual, kvout.getValue());
    assertEquals(msg1.getRequiredField(), actual.getRequiredField());

    kvout = fileReader.next();
    assertEquals(kv2.getOffset(), kvout.getOffset());
    assertArrayEquals(kv2.getValue(), kvout.getValue());
    actual = new UnitTestMessage();
    deserializer.deserialize(actual, kvout.getValue());
    assertEquals(msg2.getRequiredField(), actual.getRequiredField());
}
 

@SuppressWarnings("rawtypes")
public byte[] encodeMessage(TBase object) throws InstantiationException,
        IllegalAccessException, TException {
    TSerializer serializer = new TSerializer(messageProtocolFactory);
    return serializer.serialize(object);
}
 

public void run() {
    Properties properties = new Properties();
    if (mMetadataBrokerList == null || mMetadataBrokerList.isEmpty()) {
        properties.put("bootstrap.servers", "localhost:9092");
    } else {
        properties.put("bootstrap.severs", mMetadataBrokerList);
    }
    properties.put("value.serializer", ByteArraySerializer.class);
    properties.put("key.serializer", StringSerializer.class);
    properties.put("acks", "1");

    Producer<String, byte[]> producer = new KafkaProducer<>(properties);

    TProtocolFactory protocol = null;
    if(mType.equals("json")) {
        protocol = new TSimpleJSONProtocol.Factory();
    } else if (mType.equals("binary")) {
        protocol = new TBinaryProtocol.Factory();
    } else {
        throw new RuntimeException("Undefined message encoding type: " + mType);
    }

    TSerializer serializer = new TSerializer(protocol);
    for (int i = 0; i < mNumMessages; ++i) {
        long time = (System.currentTimeMillis() - mTimeshift * 1000L) * 1000000L + i;
        TestMessage testMessage = new TestMessage(time,
                                                  "some_value_" + i);
        if (i % 2 == 0) {
            testMessage.setEnumField(TestEnum.SOME_VALUE);
        } else {
            testMessage.setEnumField(TestEnum.SOME_OTHER_VALUE);
        }
        byte[] bytes;
        try {
            bytes = serializer.serialize(testMessage);
        } catch(TException e) {
            throw new RuntimeException("Failed to serialize message " + testMessage, e);
        }
        ProducerRecord<String, byte[]> data = new ProducerRecord<>(mTopic, Integer.toString(i), bytes);
        producer.send(data);
    }
    producer.close();
}
 

void handleStats(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { 
  if (!Constants.API_ENDPOINT_DIRECTORY_STATS_INTERNAL.equals(target)) {
    return;
  }
  
  long nano = System.nanoTime();

  baseRequest.setHandled(true);

  //
  // Read DirectoryRequests from stdin
  //
  
  BufferedReader br = new BufferedReader(request.getReader());
  
  while (true) {
    String line = br.readLine();
    
    if (null == line) {
      break;
    }
    
    byte[] raw = OrderPreservingBase64.decode(line.getBytes(StandardCharsets.US_ASCII));

    // Extract DirectoryStatsRequest
    TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
    DirectoryStatsRequest req = new DirectoryStatsRequest();
    
    try {
      deser.deserialize(req, raw);
      DirectoryStatsResponse resp = stats(req);

      response.setContentType("text/plain");
      OutputStream out = response.getOutputStream();
            
      TSerializer ser = new TSerializer(new TCompactProtocol.Factory());
      byte[] data = ser.serialize(resp);
      
      OrderPreservingBase64.encodeToStream(data, out);
      
      out.write('\r');
      out.write('\n');
    } catch (TException te) {
      throw new IOException(te);
    }            
  }
}
 

public static void main(String[] args) throws Exception {
  
  boolean dump = false;
  boolean decrypt = false;
  
  if ("dump".equals(args[0])) {
    dump = true;
  } else if ("decrypt".equals(args[0])) {
    decrypt = true;
  }
  
  KeyStore ks = new OSSKeyStore(System.getProperty("oss.master"));
  
  byte[] key = ks.decodeKey(args[1]);
  
  BufferedReader br = new BufferedReader(new FileReader(args[2]));
  
  TSerializer ser = new TSerializer(new TCompactProtocol.Factory());
  
  while(true) {
    String line = br.readLine();
    
    if (null == line) {
      break;
    }
    
    LoggingEvent event = LogUtil.unwrapLog(key, line);
    
    if (null == event) {
      continue;
    }
    
    if (dump) {
      System.out.println(event);
    } else if (decrypt) {
      byte[] serialized = ser.serialize(event);
      
      OrderPreservingBase64.encodeToStream(serialized, System.out);
      System.out.println();
    }
  }
  
  br.close();
}
 

private void sendDataMessage(KafkaDataMessage msg) throws IOException {    
  AtomicLong dms = this.dataMessagesSize.get();
  List<KeyedMessage<byte[], byte[]>> msglist = this.dataMessages.get();
  
  if (null != msg) {
    //
    // Build key
    //
    
    byte[] bytes = new byte[16];
    
    GTSHelper.fillGTSIds(bytes, 0, msg.getClassId(), msg.getLabelsId());

    //ByteBuffer bb = ByteBuffer.wrap(new byte[16]).order(ByteOrder.BIG_ENDIAN);
    //bb.putLong(encoder.getClassId());
    //bb.putLong(encoder.getLabelsId());

    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
         
    byte[] msgbytes = null;
    
    try {
      msgbytes = serializer.serialize(msg);
    } catch (TException te) {
      throw new IOException(te);
    }
    
    //
    // Encrypt value if the AES key is defined
    //
      
    if (null != this.aesDataKey) {
      msgbytes = CryptoUtils.wrap(this.aesDataKey, msgbytes);               
    }
      
    //
    // Compute MAC if the SipHash key is defined
    //
      
    if (null != this.siphashDataKey) {
      msgbytes = CryptoUtils.addMAC(this.siphashDataKey, msgbytes);
    }
      
    //KeyedMessage<byte[], byte[]> message = new KeyedMessage<byte[], byte[]>(this.dataTopic, bb.array(), msgbytes);
    KeyedMessage<byte[], byte[]> message = new KeyedMessage<byte[], byte[]>(this.dataTopic, bytes, msgbytes);
    msglist.add(message);
    //this.dataMessagesSize.get().addAndGet(bb.array().length + msgbytes.length);      
    dms.addAndGet(bytes.length + msgbytes.length);      
    
    Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_INGRESS_KAFKA_DATA_MESSAGES, Sensision.EMPTY_LABELS, 1);
  }

  if (msglist.size() > 0 && (null == msg || dms.get() > DATA_MESSAGES_THRESHOLD)) {
    Producer<byte[],byte[]> producer = getDataProducer();
    //this.dataProducer.send(msglist);
    try {

      //
      // How long it takes to send messages to Kafka
      //

      long nano = System.nanoTime();

      producer.send(msglist);

      nano = System.nanoTime() - nano;
      Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_INGRESS_KAFKA_DATA_PRODUCER_SEND, Sensision.EMPTY_LABELS, nano);

    } catch (Throwable t) {
      throw t;
    } finally {
      recycleDataProducer(producer);
    }
    Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_INGRESS_KAFKA_DATA_SEND, Sensision.EMPTY_LABELS, 1);
    msglist.clear();
    dms.set(0L);          
  }        
}
 

@Test
  public static void testSerialization()
      throws TException {
    BrokerRequest req = new BrokerRequest();

    // Populate Query Type
    QueryType type = new QueryType();
    type.setHasAggregation(true);
    type.setHasFilter(true);
    type.setHasSelection(true);
    type.setHasGroup_by(true);
    req.setQueryType(type);

    // Populate Query source
    QuerySource s = new QuerySource();
    s.setTableName("dummy");
    req.setQuerySource(s);

    req.setDuration("dummy");
    req.setTimeInterval("dummy");

    //Populate Group-By
    GroupBy groupBy = new GroupBy();
    List<String> columns = new ArrayList<String>();
    columns.add("dummy1");
    columns.add("dummy2");
    groupBy.setColumns(columns);
    groupBy.setTopN(100);
    req.setGroupBy(groupBy);

    //Populate Selections
    Selection sel = new Selection();
    sel.setSize(1);
    SelectionSort s2 = new SelectionSort();
    s2.setColumn("dummy1");
    s2.setIsAsc(true);
    sel.addToSelectionSortSequence(s2);
    sel.addToSelectionColumns("dummy1");
    req.setSelections(sel);

    //Populate FilterQuery
    FilterQuery q1 = new FilterQuery();
    q1.setId(1);
    q1.setColumn("dummy1");
    q1.addToValue("dummy1");
    q1.addToNestedFilterQueryIds(2);
    q1.setOperator(FilterOperator.AND);
    FilterQuery q2 = new FilterQuery();
    q2.setId(2);
    q2.setColumn("dummy2");
    q2.addToValue("dummy2");
    q2.setOperator(FilterOperator.AND);

    FilterQueryMap map = new FilterQueryMap();
    map.putToFilterQueryMap(1, q1);
    map.putToFilterQueryMap(2, q2);
    req.setFilterQuery(q1);
    req.setFilterSubQueryMap(map);

    //Populate Aggregations
    AggregationInfo agg = new AggregationInfo();
    agg.setAggregationType("dummy1");
    agg.putToAggregationParams("key1", "dummy1");
    req.addToAggregationsInfo(agg);

    TSerializer normalSerializer = new TSerializer();
    TSerializer compactSerializer = new TSerializer(new TCompactProtocol.Factory());
    normalSerializer.serialize(req);
    compactSerializer.serialize(req);

//    int numRequests = 100000;
//    TimerContext t = MetricsHelper.startTimer();
//    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
//    //TSerializer serializer = new TSerializer();
//    //Compact : Size 183 , Serialization Latency : 0.03361ms
//    // Normal : Size 385 , Serialization Latency : 0.01144ms
//
//    for (int i = 0; i < numRequests; i++) {
//      try {
//        serializer.serialize(req);
//        //System.out.println(s3.length);
//        //break;
//      } catch (TException e) {
//        e.printStackTrace();
//      }
//    }
//    t.stop();
//    System.out.println("Latency is :" + (t.getLatencyMs() / (float) numRequests));
  }
 

public static final String secure(String key, String script) throws WarpScriptException {
  SecureScript sscript = new SecureScript();
  sscript.setTimestamp(System.currentTimeMillis());
  sscript.setKey(key);

  byte[] scriptBytes = script.getBytes(StandardCharsets.UTF_8);
  
  // Check if we should compress the script or not
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  
  boolean compress = false;
  
  try {
    GZIPOutputStream gzos = new GZIPOutputStream(baos);
    gzos.write(scriptBytes);
    gzos.close();
    byte[] gzipped = baos.toByteArray();
    if (gzipped.length < scriptBytes.length) {
      compress = true;
      scriptBytes = gzipped;
    }
  } catch (IOException ioe) {              
  }
  
  sscript.setCompressed(compress);
  sscript.setScript(scriptBytes);
  
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  
  try {
    byte[] serialized = serializer.serialize(sscript);
    // TODO(hbs): encrypt script
    
    synchronized(SECURE.class) {
      if (null == aesKey) {
        try {
          aesKey = WarpDist.getKeyStore().getKey(KeyStore.AES_SECURESCRIPTS);
        } catch (Throwable t) {
          // Catch NoClassDefFoundError
        }
      }
    }
    
    if (null == aesKey) {
      throw new WarpScriptException("Missing secure script encryption key.");
    }
    
    byte[] wrapped = CryptoUtils.wrap(aesKey, serialized);
    
    String encoded = new String(OrderPreservingBase64.encode(wrapped), StandardCharsets.US_ASCII);
    
    return encoded;
  } catch (TException te) {
    throw new WarpScriptException("Unable to secure script.", te);
  }

}
 

public byte[] toBytes() throws IOException {
  HyperLogLogPlusParameters params = new HyperLogLogPlusParameters();
  
  params.setInitTime(this.initTime);
  params.setP((byte) this.p);
  params.setPprime((byte) this.pprime);
  params.setSparse(Format.SPARSE == this.format);
  
  if (null != this.key) {
    params.setKey(this.key);
  }
  
  if (Format.SPARSE == this.format) {      
    // Trigger a merge
    merge();
    // Output the sparse list size
    params.setSparseListLen(sparse_list_len);
    params.setSparseList(sparse_list);
  } else {
    // Output the registers
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    GZIPOutputStream gzos = new GZIPOutputStream(baos);
    gzos.write(this.M);
    gzos.close();
    
    byte[] gzipped = baos.toByteArray();
    
    if (gzipped.length < this.M.length) {
      params.setRegisters(gzipped);
      params.setGzipped(true);
    } else {
      params.setRegisters(this.M);
      params.setGzipped(false);
    }
  }
  
  try {
    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
    byte[] ser = serializer.serialize(params);
    return ser;
  } catch (TException te) {
    throw new IOException(te);
  }
}