com.datatorrent.api.StreamCodec Java Examples

private HashMap.SimpleEntry<String, ComponentContextPair<Stream, StreamContext>> deployBufferServerPublisher(
    String connIdentifier, StreamCodec<?> streamCodec, long finishedWindowId, int queueCapacity,
    OperatorDeployInfo.OutputDeployInfo nodi)
    throws UnknownHostException
{
  String sinkIdentifier = "tcp://".concat(nodi.bufferServerHost).concat(":").concat(String.valueOf(nodi.bufferServerPort)).concat("/").concat(connIdentifier);

  StreamContext bssc = new StreamContext(nodi.declaredStreamId);
  bssc.setPortId(nodi.portName);
  bssc.setSourceId(connIdentifier);
  bssc.setSinkId(sinkIdentifier);
  bssc.setFinishedWindowId(finishedWindowId);
  bssc.put(StreamContext.CODEC, streamCodec);
  bssc.put(StreamContext.EVENT_LOOP, eventloop);
  bssc.setBufferServerAddress(InetSocketAddress.createUnresolved(nodi.bufferServerHost, nodi.bufferServerPort));
  bssc.put(StreamContext.BUFFER_SERVER_TOKEN, nodi.bufferServerToken);
  InetAddress inetAddress = bssc.getBufferServerAddress().getAddress();
  if (inetAddress != null && NetUtils.isLocalAddress(inetAddress)) {
    bssc.setBufferServerAddress(new InetSocketAddress(InetAddress.getByName(null), nodi.bufferServerPort));
  }

  Stream publisher = fastPublisherSubscriber ? new FastPublisher(connIdentifier, queueCapacity * 256) : new BufferServerPublisher(connIdentifier, queueCapacity);
  return new HashMap.SimpleEntry<>(sinkIdentifier, new ComponentContextPair<>(publisher, bssc));
}
 

public boolean shouldCaptureEvent(T o)
{
  for (Entry<InputPortMeta, Collection<PartitionKeys>> entry : inputPortToPartitionMap.entrySet()) {
    StreamCodec<Object> codec = codecsToMerge.get(entry.getKey());
    Collection<PartitionKeys> partitionKeysList = entry.getValue();

    for (PartitionKeys keys : partitionKeysList) {
      if ( keys.partitions != null && keys.partitions.contains(keys.mask & codec.getPartition(o))) {
        // Then at least one of the partitions is getting this event
        // So send the event to persist operator
        return true;
      }
    }
  }

  return false;
}
 

/**
 *
 * @param serde
 * @param partitions
 * @param mask
 * @param output
 */
public PartitionAwareSink(StreamCodec<T> serde, Set<Integer> partitions, int mask, Sink<T> output)
{
  this.serde = serde;
  this.partitions = partitions;
  this.output = output;
  this.mask = mask;
}
 

private void checkPresentStreamCodecInfo(Map<Integer, StreamCodec<?>> streamCodecs, String id,
    Integer streamCodecIdentifier, StreamCodec<?> streamCodecInfo)
{
  StreamCodec<?> opStreamCodecInfo = streamCodecs.get(streamCodecIdentifier);
  Assert.assertNotNull("stream codec info null " + id, opStreamCodecInfo);
  Assert.assertEquals("stream codec not same " + id, opStreamCodecInfo, streamCodecInfo);
}
 

private void checkPresentStreamCodec(LogicalPlan.OperatorMeta operatorMeta, Operator.InputPort<?> inputPort,
    Map<Integer, StreamCodec<?>> streamCodecs,
    String id, PhysicalPlan plan)
{
  StreamCodec<?> streamCodec = operatorMeta.getMeta(inputPort).getStreamCodec();
  Assert.assertTrue("stream codec identifier not present" + id, isStrCodecPresent(streamCodec, plan));
  Integer streamCodecIdentifier = plan.getStreamCodecIdentifier(streamCodec);
  checkPresentStreamCodecInfo(streamCodecs, id, streamCodecIdentifier, streamCodec);
}
 

/**
 * Returns the streamcodec for the streams
 * @param isStream1data Specifies whether the codec needs for stream1 or stream2.
 * @return the object of JoinStreamCodec
 */
private StreamCodec<Object> getInnerJoinStreamCodec(boolean isStream1data)
{
  if (isStream1data) {
    return new JoinStreamCodec(getLeftKeyExpression());
  }
  return new JoinStreamCodec(getRightKeyExpression());
}
 

public Integer getStreamCodecIdentifier(StreamCodec<?> streamCodecInfo)
{
  Integer id;
  synchronized (streamCodecIdentifiers) {
    id = streamCodecIdentifiers.get(streamCodecInfo);
    if (id == null) {
      id = strCodecIdSequence.incrementAndGet();
      streamCodecIdentifiers.put(streamCodecInfo, id);
    }
  }
  return id;
}
 

private void addStreamCodec(InputPortMeta sinkToPersistPortMeta, InputPort<?> port)
{
  StreamCodec<Object> inputStreamCodec = (StreamCodec<Object>)sinkToPersistPortMeta.getStreamCodec();
  if (inputStreamCodec != null) {
    Map<InputPortMeta, StreamCodec<Object>> codecs = new HashMap<>();
    codecs.put(sinkToPersistPortMeta, inputStreamCodec);
    InputPortMeta persistOperatorPortMeta = assertGetPortMeta(port);
    StreamCodec<Object> specifiedCodecForPersistOperator = (StreamCodec<Object>)persistOperatorPortMeta.getStreamCodec();
    StreamCodecWrapperForPersistance<Object> codec = new StreamCodecWrapperForPersistance<>(codecs, specifiedCodecForPersistOperator);
    persistOperatorPortMeta.setStreamCodec(codec);
  }
}
 

void setStreamCodec(StreamCodec<?> streamCodec)
{
  if (streamCodec != null) {
    StreamCodec<?> oldStreamCodec = attributes.put(STREAM_CODEC, streamCodec);
    if (oldStreamCodec != null && oldStreamCodec != streamCodec) { // once input port codec is set, it is not expected that it will be changed.
      LOG.warn("Operator {} input port {} stream codec was changed from {} to {}", getOperatorMeta().getName(), getPortName(), oldStreamCodec, streamCodec);
    }
  }
}
 

@Override
public StreamCodec<byte[]> getStreamCodec()
{
  if (ValidationToFile.this.streamCodec == null) {
    return super.getStreamCodec();
  } else {
    return streamCodec;
  }
}
 

public StreamCodec<Object> getSpecifiedStreamCodec()
{
  if (specifiedStreamCodec == null) {
    specifiedStreamCodec = new DefaultKryoStreamCodec();
  }
  return specifiedStreamCodec;
}
 

@Override
public StreamCodec<Map<K, V>> getStreamCodec()
{
  StreamCodec<Map<K, V>> streamCodec = AbstractBaseNOperatorMap.this.getStreamCodec();
  if (streamCodec == null) {
    return super.getStreamCodec();
  } else {
    return streamCodec;
  }
}
 

@Override
@SuppressWarnings("unchecked")
public void setup(StreamContext context)
{
  StreamCodec<?> codec = context.get(StreamContext.CODEC);
  if (codec == null) {
    statefulSerde = ((StatefulStreamCodec<Object>)StreamContext.CODEC.defaultValue).newInstance();
  } else if (codec instanceof StatefulStreamCodec) {
    statefulSerde = ((StatefulStreamCodec<Object>)codec).newInstance();
  } else {
    serde = (StreamCodec<Object>)codec;
  }
}
 

public SweepableReservoir acquireReservoirForPersistStream(String id, int capacity, StreamCodec<?> streamCodec)
{
  BufferReservoir r = reservoirMap.get(id);
  if (r == null) {
    reservoirMap.put(id, r = new BufferReservoirForPersistStream(capacity, (StreamCodecWrapperForPersistance<Object>)streamCodec));
    BufferReservoir[] newReservoirs = new BufferReservoir[reservoirs.length + 1];
    newReservoirs[reservoirs.length] = r;
    for (int i = reservoirs.length; i-- > 0;) {
      newReservoirs[i] = reservoirs[i];
    }
    reservoirs = newReservoirs;
  }

  return r;
}
 

@Override
@SuppressWarnings("unchecked")
public void setup(StreamContext context)
{
  StreamCodec<?> codec = context.get(StreamContext.CODEC);
  if (codec == null) {
    statefulSerde = ((StatefulStreamCodec<Object>)StreamContext.CODEC.defaultValue).newInstance();
  } else if (codec instanceof StatefulStreamCodec) {
    statefulSerde = ((StatefulStreamCodec<Object>)codec).newInstance();
  } else {
    serde = (StreamCodec<Object>)codec;
  }
  baseSeconds = context.getFinishedWindowId() & 0xffffffff00000000L;
}
 

/**
 * Set StreamCodec used for partitioning.
 */
@Override
public StreamCodec<KeyValPair<K, V>> getStreamCodec()
{
  return getKeyValPairStreamCodec();
}
 

/**
 * Set StreamCodec used for partitioning.
 */
@Override
public StreamCodec<KeyValPair<K, V>> getStreamCodec()
{
  return getKeyValPairStreamCodec();
}
 

/**
 * A codec to enable partitioning to be done by key
 */
public StreamCodec<KeyValPair<K, V>> getKeyValPairStreamCodec()
{
  return new DefaultPartitionCodec<K, V>();
}
 

@Override
public StreamCodec<KeyValPair<K, V>> getStreamCodec()
{
  return getKeyValPairStreamCodec();
}
 

/**
 * Stream codec used for partitioning.
 */
@Override
public StreamCodec<KeyValPair<K, V>> getStreamCodec()
{
  return getKeyValPairStreamCodec();
}
 

@Override
protected StreamCodec<Map<String, DimensionObject<String>>> getStreamCodec()
{
  return new LogstreamTopNStreamCodec();
}
 

/**
 * Set StreamCodec used for partitioning.
 */
@Override
public StreamCodec<KeyValPair<K, V>> getStreamCodec()
{
  return getKeyValPairStreamCodec();
}
 

/**
 * Set StreamCodec used for partitioning.
 */
@Override
public StreamCodec<KeyValPair<K, V>> getStreamCodec()
{
  return getKeyValPairStreamCodec();
}
 

public StreamCodec getStreamCodec()
{
  return streamCodec;
}
 

public void setStreamCodec(StreamCodec streamCodec)
{
  this.streamCodec = streamCodec;
}
 

@Override
public StreamCodec<Object> getStreamCodec()
{
  return getInnerJoinStreamCodec(true);
}
 

@Override
public StreamCodec<Object> getStreamCodec()
{
  return getInnerJoinStreamCodec(false);
}
 

@Override
public StreamCodec<Object> getStreamCodec()
{
  return streamCodec;
}
 

protected StreamCodec<Object> getDeduperStreamCodec()
{
  return new DeduperStreamCodec(keyExpression);
}
 

@Override
public StreamCodec<Object> getStreamCodec()
{
  return streamCodec;
}