org.apache.storm.generated.GlobalStreamId Java Examples

public static Map<String, Set<String>> getAdjacencyMap(StormTopology topology,
                                                       boolean removeSystemComponent) {
    Map<String, Set<String>> adjacencyMap = new HashMap<>();

    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String boltName = entry.getKey();
        Map<GlobalStreamId, Grouping> inputs = entry.getValue().get_common().get_inputs();
        for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) {
            String inputComponentId = input.getKey().get_componentId();
            Set<String> components = adjacencyMap.containsKey(inputComponentId)
                    ? adjacencyMap.get(inputComponentId) : new HashSet<String>();
            components.add(boltName);
            components = removeSystemComponent ? removeSystemComponents(components)
                    : components;
            if (!removeSystemComponent || !isSystemComponent(inputComponentId)) {
                adjacencyMap.put(inputComponentId, components);
            }
        }
    }

    return adjacencyMap;
}
 

@Test
public void aggregate() throws Exception {
	beansXml = "<breeze:topology id='aggregate'>" +
			"<breeze:spout id='iterator' beanType='java.util.Iterator' signature='next()' outputFields='x'/>" +
			"<breeze:spout id='queue' beanType='java.util.Queue' signature='poll()' outputFields='x'/>" +
			"<breeze:bolt id='collector' beanType='org.slf4j.Logger' signature='info(x)'/>" +
			"</breeze:topology>";
	refresh();

	StormTopology topology = getBean("aggregate", StormTopology.class);

	Bolt collector = topology.get_bolts().get("collector");
	Map<GlobalStreamId, Grouping> inputs = collector.get_common().get_inputs();
	assertEquals("input count", 2, inputs.size());
	assertNotNull("iterator grouping", inputs.get(new GlobalStreamId("iterator", "default")));
	assertNotNull("queue grouping", inputs.get(new GlobalStreamId("queue", "default")));
}
 

@Override
public void execute(Tuple tuple) {
    List<Object> id = tuple.select(_idFields);
    GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId());
    if (!_pending.containsKey(id)) {
        _pending.put(id, new HashMap<GlobalStreamId, Tuple>());
    }
    Map<GlobalStreamId, Tuple> parts = _pending.get(id);
    if (parts.containsKey(streamId)) {
        throw new RuntimeException("Received same side of single join twice");
    }
    parts.put(streamId, tuple);
    if (parts.size() == _numSources) {
        _pending.remove(id);
        List<Object> joinResult = new ArrayList<Object>();
        for (String outField : _outFields) {
            GlobalStreamId loc = _fieldLocations.get(outField);
            joinResult.add(parts.get(loc).getValueByField(outField));
        }
        _collector.emit(new ArrayList<Tuple>(parts.values()), joinResult);

        for (Tuple part : parts.values()) {
            _collector.ack(part);
        }
    }
}
 

public static Map<String, Set<String>> getAdjacencyMap(StormTopology topology,
                                                       boolean removeSystemComponent)
throws Exception {
    Map<String, Set<String>> adjacencyMap = new HashMap<>();

    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String boltName = entry.getKey();
        Map<GlobalStreamId, Grouping> inputs = entry.getValue().get_common().get_inputs();
        for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) {
            String inputComponentId = input.getKey().get_componentId();
            Set<String> components = adjacencyMap.containsKey(inputComponentId)
                    ? adjacencyMap.get(inputComponentId) : new HashSet<String>();
            components.add(boltName);
            components = removeSystemComponent ? removeSystemComponents(components)
                    : components;
            if (!removeSystemComponent || !isSystemComponent(inputComponentId)) {
                adjacencyMap.put(inputComponentId, components);
            }
        }
    }

    return adjacencyMap;
}
 

public static Set<String> getTerminalUserBoltNames(StormTopology topology) {
    Set<String> terminalBolts = new HashSet<>();
    Set<String> inputs = new HashSet<>();
    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String name = entry.getKey();
        Set<GlobalStreamId> inputsForBolt = entry.getValue().get_common().get_inputs().keySet();
        if (!isSystemComponent(name)) {
            for (GlobalStreamId streamId : inputsForBolt) {
                inputs.add(streamId.get_componentId());
            }
        }
    }

    for (String boltName : topology.get_bolts().keySet()) {
        if (!isSystemComponent(boltName) && !inputs.contains(boltName)) {
            terminalBolts.add(boltName);
        }
    }

    return terminalBolts;
}
 

@Override
public void execute(Tuple tuple) {
  List<Object> id = tuple.select(_idFields);
  GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId());
  if (!_pending.containsKey(id)) {
    _pending.put(id, new HashMap<GlobalStreamId, Tuple>());
  }
  Map<GlobalStreamId, Tuple> parts = _pending.get(id);
  if (parts.containsKey(streamId))
    throw new RuntimeException("Received same side of single join twice");
  parts.put(streamId, tuple);
  if (parts.size() == _numSources) {
    _pending.remove(id);
    List<Object> joinResult = new ArrayList<Object>();
    for (String outField : _outFields) {
      GlobalStreamId loc = _fieldLocations.get(outField);
      joinResult.add(parts.get(loc).getValueByField(outField));
    }
    _collector.emit(new ArrayList<Tuple>(parts.values()), joinResult);

    for (Tuple part : parts.values()) {
      _collector.ack(part);
    }
  }
}
 

public static Set<String> getTerminalUserBoltNames(StormTopology topology) {
    Set<String> terminalBolts = new HashSet<>();
    Set<String> inputs = new HashSet<>();
    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String name = entry.getKey();
        Set<GlobalStreamId> inputsForBolt = entry.getValue().get_common().get_inputs().keySet();
        if (!isSystemComponent(name)) {
            for (GlobalStreamId streamId : inputsForBolt) {
                inputs.add(streamId.get_componentId());
            }
        }
    }

    for (String boltName : topology.get_bolts().keySet()) {
        if (!isSystemComponent(boltName) && !inputs.contains(boltName)) {
            terminalBolts.add(boltName);
        }
    }

    return terminalBolts;
}
 

/**
 * Creates a new WatermarkEventGenerator.
 *
 * @param windowManager The window manager this generator will submit watermark events to
 * @param intervalMs The generator will check if it should generate a watermark event
 * with this interval
 * @param eventTsLagMs The max allowed lag behind the last watermark event before an
 * event is considered late
 * @param inputStreams The input streams this generator is expected to handle
 */
public WaterMarkEventGenerator(WindowManager<T> windowManager, int intervalMs,
                               int eventTsLagMs, Set<GlobalStreamId> inputStreams) {
  this.windowManager = windowManager;
  streamToTs = new ConcurrentHashMap<>();

  ThreadFactory threadFactory = new ThreadFactory() {
    private int threadId;

    @Override
    public Thread newThread(Runnable r) {
      Thread thread = new Thread(r);
      thread.setName(String.format("watermark-event-generator-%d", threadId++));
      thread.setDaemon(true);
      return thread;
    }
  };
  executorService = Executors.newSingleThreadScheduledExecutor(threadFactory);

  this.interval = intervalMs;
  this.eventTsLag = eventTsLagMs;
  this.inputStreams = inputStreams;
}
 

@Override
public void prepare(org.apache.heron.api.topology.TopologyContext context,
                    String component, String streamId,
                    List<Integer> targetTasks) {
  TopologyContext c = new TopologyContext(context);
  GlobalStreamId g = new GlobalStreamId(component, streamId);
  delegate.prepare(c, g, targetTasks);
}
 

@Override
public void prepare(WorkerTopologyContext context, GlobalStreamId stream,
        List<Integer> targetTasks) {
    this.targetTask = targetTasks;
    partitioner = new URLPartitioner();
    if (StringUtils.isNotBlank(partitionMode)) {
        Map<String, String> conf = new HashMap<>();
        conf.put(Constants.PARTITION_MODEParamName, partitionMode);
        partitioner.configure(conf);
    }
}
 

@Override
public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
    choices = new ArrayList<>(targetTasks.size());
    allTasks = new ArrayList<>(targetTasks.size());
    for (Integer i : targetTasks) {
        choices.add(Arrays.asList(i));
        allTasks.add(i);
    }
}
 

@Override
public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
    _fieldLocations = new HashMap<String, GlobalStreamId>();
    _collector = collector;
    int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
    _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback());
    _numSources = context.getThisSources().size();
    Set<String> idFields = null;
    for (GlobalStreamId source : context.getThisSources().keySet()) {
        Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId());
        Set<String> setFields = new HashSet<String>(fields.toList());
        if (idFields == null) {
            idFields = setFields;
        } else {
            idFields.retainAll(setFields);
        }

        for (String outfield : _outFields) {
            for (String sourcefield : fields) {
                if (outfield.equals(sourcefield)) {
                    _fieldLocations.put(outfield, source);
                }
            }
        }
    }
    _idFields = new Fields(new ArrayList<String>(idFields));

    if (_fieldLocations.size() != _outFields.size()) {
        throw new RuntimeException("Cannot find all outfields among sources");
    }
}
 

@Override
public void prepare(
    WorkerTopologyContext context,
    GlobalStreamId stream,
    List<Integer> targetTasks) {
  this.taskIds = targetTasks;
}
 

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
  _fieldLocations = new HashMap<String, GlobalStreamId>();
  _collector = collector;
  int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
  _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback());
  _numSources = context.getThisSources().size();
  Set<String> idFields = null;
  for (GlobalStreamId source : context.getThisSources().keySet()) {
    Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId());
    Set<String> setFields = new HashSet<String>(fields.toList());
    if (idFields == null)
      idFields = setFields;
    else
      idFields.retainAll(setFields);

    for (String outfield : _outFields) {
      for (String sourcefield : fields) {
        if (outfield.equals(sourcefield)) {
          _fieldLocations.put(outfield, source);
        }
      }
    }
  }
  _idFields = new Fields(new ArrayList<String>(idFields));

  if (_fieldLocations.size() != _outFields.size()) {
    throw new RuntimeException("Cannot find all outfields among sources");
  }
}
 

private Set<GlobalStreamId> getComponentStreams(TopologyContext context) {
  Set<GlobalStreamId> streams = new HashSet<>();
  for (GlobalStreamId streamId : context.getThisSources().keySet()) {
    if (!streamId.get_streamId().equals(CheckpointSpout.CHECKPOINT_STREAM_ID)) {
      streams.add(streamId);
    }
  }
  return streams;
}
 

/**
 * Gets the declared inputs to this component.
 * todo the return type of this method should be {@literal Map<GlobalStreamId, Grouping>}, where
 * Grouping is a thrift object. So changing it to Object, just to support windowing.
 * {@link org.apache.storm.topology.WindowedBoltExecutor} just need the {@link GlobalStreamId}
 *
 * @return A map from subscribed component/stream to the grouping subscribed with.
 */
public Map<GlobalStreamId, Object> getThisSources() {
  if (this.boltDeclarer == null) {
    throw new RuntimeException("Couldn't determine sources for this Node");
  }
  Map<GlobalStreamId, Object> sourcesMap = new HashMap<>();
  this.boltDeclarer.getGroupings().forEach(twister2BoltGrouping -> {
    GlobalStreamId globalStreamId = new GlobalStreamId();
    globalStreamId.set_componentId(twister2BoltGrouping.getComponentId());
    globalStreamId.set_streamId(twister2BoltGrouping.getStreamId());
    sourcesMap.put(globalStreamId, null);
  });
  return sourcesMap;
}
 

/**
 * Computes the min ts across all streams.
 */
private long computeWaterMarkTs() {
  long ts = 0;
  // only if some data has arrived on each input stream
  if (streamToTs.size() >= inputStreams.size()) {
    ts = Long.MAX_VALUE;
    for (Map.Entry<GlobalStreamId, Long> entry : streamToTs.entrySet()) {
      ts = Math.min(ts, entry.getValue());
    }
  }
  return ts - eventTsLag;
}
 

/**
 * Tracks the timestamp of the event in the stream, returns true if the event can
 * be considered for processing or false if its a late
 * event.
 */
public boolean track(GlobalStreamId stream, long ts) {
  Long currentVal = streamToTs.get(stream);
  if (currentVal == null || ts > currentVal) {
    streamToTs.put(stream, ts);
  }
  checkFailures();
  return ts >= lastWaterMarkTs;
}
 

@Override
public void expire(List<Object> id, Map<GlobalStreamId, Tuple> tuples) {
  for (Tuple tuple : tuples.values()) {
    _collector.fail(tuple);
  }
}
 

@Override
public GlobalStreamId getSourceGlobalStreamId() {
    throw new UnsupportedOperationException();
}
 

@Override
public void prepare(WorkerTopologyContext context, GlobalStreamId stream,
                    List<Integer> targetTasks) {

}
 

@Override
public GlobalStreamId getSourceGlobalStreamId() {
  return new GlobalStreamId(getSourceComponent(), getSourceStreamId());
}
 

@Override
public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
    this.targetTasks = targetTasks;
}
 

@Override
public void expire(List<Object> id, Map<GlobalStreamId, Tuple> tuples) {
    for (Tuple tuple : tuples.values()) {
        _collector.fail(tuple);
    }
}
 

@Test
public void build() throws Exception {
	beansXml = "<breeze:topology id='t1'>" +
			"<breeze:spout id='s1' beanType='eu.icolumbo.breeze.TestBean' signature='ping()' outputFields='feed'/>" +
			"<breeze:bolt id='b1' beanType='eu.icolumbo.breeze.TestBean' signature='echo(feed)' outputFields='replay' scatterOutput='true'/>" +
			"<breeze:bolt beanType='eu.icolumbo.breeze.TestBean' signature='drain(replay)' parallelism='2'/>" +
			"</breeze:topology>";
	refresh();

	StormTopology topology = getBean("t1", StormTopology.class);
	assertEquals("spout count", 1, topology.get_spouts_size());
	assertEquals("bolt count", 2, topology.get_bolts_size());

	SpringSpout spout = getBean("s1", SpringSpout.class);
	assertEquals("spout ID", "s1", spout.getId());
	assertEquals("spout scatter", false, spout.getScatterOutput());
	SpringBolt bolt = getBean("b1", SpringBolt.class);
	assertEquals("bolt ID", "b1", bolt.getId());
	assertEquals("bolt scatter", true, bolt.getScatterOutput());

	Map<String, SpoutSpec> topologySpouts = topology.get_spouts();
	SpoutSpec spoutSpec = topologySpouts.get("s1");
	assertNotNull("s1 spec", spoutSpec);

	Map<String, Bolt> topologyBolts = topology.get_bolts();
	Bolt boltSpec = topologyBolts.get("b1");
	assertNotNull("b1 spec", boltSpec);

	String anonymousBoltId = null;
	for (String id : topologyBolts.keySet())
		if (! "b1".equals(id))
			anonymousBoltId = id;
	assertNotNull("anonymous ID", anonymousBoltId);
	Bolt anonymousBoltSpec = topologyBolts.get(anonymousBoltId);
	assertNotNull("anonymous spec", anonymousBoltSpec);

	assertEquals("s1 parralelism", 1, spoutSpec.get_common().get_parallelism_hint());
	assertEquals("b1 parralelism", 1, boltSpec.get_common().get_parallelism_hint());
	assertEquals("second bold parrallelism", 2, anonymousBoltSpec.get_common().get_parallelism_hint());

	Map<GlobalStreamId,Grouping> boltInputs = boltSpec.get_common().get_inputs();
	assertEquals("input size", 1, boltInputs.size());
	GlobalStreamId streamId = boltInputs.keySet().iterator().next();
	assertEquals("input component id", "s1", streamId.get_componentId());
	assertEquals("input stream id", "default", streamId.get_streamId());
}
 

/**
 * Tells the stream grouping at runtime the tasks in the target bolt.
 * This information should be used in chooseTasks to determine the target tasks.
 * <p>
 * It also tells the grouping the metadata on the stream this grouping will be used on.
 */
void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks);
 

/**
 * Tells the stream grouping at runtime the tasks in the target bolt.
 * This information should be used in chooseTasks to determine the target tasks.
 * <p>
 * It also tells the grouping the metadata on the stream this grouping will be used on.
 */
void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks);
 

/**
 * Returns the global stream id (component + stream) of this tuple.
 */
GlobalStreamId getSourceGlobalStreamId();