backtype.storm.generated.GlobalStreamId Java Examples

public static Set<Integer> getDownstreamTasks(String componentId, TopologyContext context) {
    Set<Integer> tasks = new HashSet<>();
    StormTopology topology = context.getRawTopology();

    Map<String, Bolt> bolts = topology.get_bolts();
    for (Entry<String, Bolt> entry : bolts.entrySet()) {
        String downstreamComponentId = entry.getKey();
        Bolt bolt = entry.getValue();
        Set<GlobalStreamId> input = bolt.get_common().get_inputs().keySet();
        for (GlobalStreamId stream : input) {
            String upstreamComponentId = stream.get_componentId();
            if (upstreamComponentId.equals(componentId)) {
                tasks.addAll(context.getComponentTasks(downstreamComponentId));
                break;
            }
        }
    }

    return tasks;
}
 

public static Set<String> getAllDownstreamComponents(String componentId, StormTopology topology,
                                                     Set<String> traversedComponents) {
    Set<String> components = new HashSet<>();
    traversedComponents.add(componentId);

    Map<String, Bolt> bolts = topology.get_bolts();
    for (Entry<String, Bolt> entry : bolts.entrySet()) {
        String downstreamComponentId = entry.getKey();
        Bolt bolt = entry.getValue();
        Set<GlobalStreamId> input = bolt.get_common().get_inputs().keySet();
        for (GlobalStreamId stream : input) {
            String upstreamComponentId = stream.get_componentId();
            if (upstreamComponentId.equals(componentId) && !traversedComponents.contains(downstreamComponentId)) {
                components.add(downstreamComponentId);
                components.addAll(getAllDownstreamComponents(downstreamComponentId, topology, traversedComponents));
                break;
            }
        }
    }

    return components;
}
 

/**
 * Gets information about who is consuming the outputs of the specified component, and how.
 *
 * @return Map from stream id to component id to the Grouping used.
 */
public Map<String, Map<String, Grouping>> getTargets(String componentId) {
    Map<String, Map<String, Grouping>> ret = new HashMap<>();
    for (String otherComponentId : getComponentIds()) {
        Map<GlobalStreamId, Grouping> inputs = getComponentCommon(otherComponentId).get_inputs();
        for (GlobalStreamId id : inputs.keySet()) {
            if (id.get_componentId().equals(componentId)) {
                Map<String, Grouping> curr = ret.get(id.get_streamId());
                if (curr == null)
                    curr = new HashMap<>();
                curr.put(otherComponentId, inputs.get(id));
                ret.put(id.get_streamId(), curr);
            }
        }
    }
    return ret;
}
 

public static Set<String> getDownstreamComponents(String componentId, StormTopology topology) {
    Set<String> components = new HashSet<>();

    Map<String, Bolt> bolts = topology.get_bolts();
    for (Entry<String, Bolt> entry : bolts.entrySet()) {
        String downstreamComponentId = entry.getKey();
        Bolt bolt = entry.getValue();
        Set<GlobalStreamId> input = bolt.get_common().get_inputs().keySet();
        for (GlobalStreamId stream : input) {
            String upstreamComponentId = stream.get_componentId();
            if (upstreamComponentId.equals(componentId)) {
                components.add(downstreamComponentId);
                break;
            }
        }
    }

    return components;
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    createState(context);
    prepare(stormConf, context, collector, keyRangeState);
    
    Map<GlobalStreamId, Grouping> sources = context.getSources(context.getThisComponentId());
    for (Map.Entry<GlobalStreamId, Grouping> entry : sources.entrySet()) {
        GlobalStreamId stream = entry.getKey();
        Grouping grouping = entry.getValue();
        Grouping._Fields groupingFields = Thrift.groupingType(grouping);
        if (Grouping._Fields.FIELDS.equals(groupingFields)) {
            Fields fields = new Fields(Thrift.fieldGrouping(grouping));
            fieldGrouping.put(stream.get_streamId(), fields);
        }
    }
    LOG.info("Source fieldgrouping streams: {}", fieldGrouping);
}
 

@Override
public void execute(Tuple tuple) {
    List<Object> id = tuple.select(_idFields);
    GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId());
    if (!_pending.containsKey(id)) {
        _pending.put(id, new HashMap<GlobalStreamId, Tuple>());
    }
    Map<GlobalStreamId, Tuple> parts = _pending.get(id);
    if (parts.containsKey(streamId))
        throw new RuntimeException("Received same side of single join twice");
    parts.put(streamId, tuple);
    if (parts.size() == _numSources) {
        _pending.remove(id);
        List<Object> joinResult = new ArrayList<Object>();
        for (String outField : _outFields) {
            GlobalStreamId loc = _fieldLocations.get(outField);
            joinResult.add(parts.get(loc).getValueByField(outField));
        }
        _collector.emit(new ArrayList<Tuple>(parts.values()), joinResult);
        
        for (Tuple part : parts.values()) {
            _collector.ack(part);
        }
    }
}
 

public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
  
  System.out.println("preparing HBaseStreamPartitioner for streamId " + stream.get_streamId());
  this.targetTasks = targetTasks;
  this.targetTasksSize = this.targetTasks.size();

  Configuration conf = HBaseConfiguration.create();
  try {
    hTable = new HTable(conf, tableName);
    refreshRegionInfo(tableName);

    System.out.println("regionStartKeyRegionNameMap: " + regionStartKeyRegionNameMap);

  } catch (IOException e) {
    e.printStackTrace();
  }

}
 

@Override
public void initState(T state) {
    if (stateInitialized) {
        LOG.warn("State is already initialized. Ignoring initState");
        return;
    }
    statefulWindowedBolt.initState((T) state);
    // query the streamState for each input task stream and compute recoveryStates
    for (GlobalStreamId streamId : topologyContext.getThisSources().keySet()) {
        for (int taskId : topologyContext.getComponentTasks(streamId.get_componentId())) {
            WindowState windowState = streamState.get(new TaskStream(taskId, streamId));
            if (windowState != null) {
                recoveryStates.put(new TaskStream(taskId, streamId), windowState);
            }
        }
    }
    LOG.debug("recoveryStates {}", recoveryStates);
    stateInitialized = true;
    start();
}
 

protected void initCommon(String id, IComponent component, Number parallelism) throws IllegalArgumentException {
    ComponentCommon common = new ComponentCommon();
    common.set_inputs(new HashMap<GlobalStreamId, Grouping>());
    if (parallelism != null) {
        int dop = parallelism.intValue();
        if(dop < 1) {
            throw new IllegalArgumentException("Parallelism must be positive.");
        }
        common.set_parallelism_hint(dop);
    } else {
        common.set_parallelism_hint(1);
    }
    Map conf = component.getComponentConfiguration();
    if(conf!=null) common.set_json_conf(JSONValue.toJSONString(conf));
    _commons.put(id, common);
}
 

@SuppressWarnings("rawtypes")
@Override
void prepare(Map conf, TopologyContext context) {
	// use TTL and maxSize from config if they were not set explicitly using the constructor (implicit way of doing this...)
	if(TTL == 29) TTL = conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC) == null ? TTL : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC)).intValue();
	if(maxSize == 256) maxSize = conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE) == null ? maxSize : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE)).intValue();
	history = new History(this);
	
	// IF NO grouping was set THEN select the first grouping registered for the spout as the grouping used within the Spout (usually a good guess)
	if(groupBy == null){
		Map<GlobalStreamId, Grouping> sources = context.getSources(context.getThisComponentId());
		for(GlobalStreamId id : sources.keySet()){
			Grouping grouping = sources.get(id);
			this.groupBy = new Fields(grouping.get_fields());
			break;
		}
	}
	
	// prepare the selector and operation
	try {
		batcher.prepare(conf);
		operation.prepare(conf, context);
	} catch (Exception e) {
		logger.error("Unable to preapre the Selector or Operation", e);
	}
}
 

private static boolean hasDownstreamComponent(StormTopology topology, String outComponentId, String outStreamId) {
    boolean ret = false;
    for (String componentId : ThriftTopologyUtils.getComponentIds(topology)) {
        ComponentCommon componentCommon = Utils.getComponentCommon(topology, componentId);
        Set<GlobalStreamId> inputs = componentCommon.get_inputs().keySet();
        for (GlobalStreamId input : inputs) {
            if (input.get_componentId().equals(outComponentId) && input.get_streamId().equals(outStreamId))
                return true;
        }
    }
    return ret;
}
 

public WaterMarkEventGenerator(WindowManager<T> windowManager, int interval,
                               int eventTsLag, Set<GlobalStreamId> inputStreams) {
    this.windowManager = windowManager;
    streamToTs = new ConcurrentHashMap<>();
    executorService = Executors.newSingleThreadScheduledExecutor();
    this.interval = interval;
    this.eventTsLag = eventTsLag;
    this.inputStreams = inputStreams;
}
 

public MkCustomGrouper(TopologyContext context, CustomStreamGrouping grouping,
                       GlobalStreamId stream, List<Integer> targetTask, int myTaskId) {
    this.myTaskId = myTaskId;
    this.grouping = grouping;
    this.grouping.prepare(context, stream, targetTask);

}
 

private Fields getSourceOutputFields(TopologyContext context, String sourceStream) {
    for (GlobalStreamId g : context.getThisSources().keySet()) {
        if (g.get_streamId().equals(sourceStream)) {
            return context.getComponentOutputFields(g);
        }
    }
    throw new RuntimeException("Could not find fields for source stream " + sourceStream);
}
 

@Override
public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> tasks) {
    List<Integer> sourceTasks = new ArrayList<Integer>(context.getComponentTasks(stream.get_componentId()));
    Collections.sort(sourceTasks);
    if (sourceTasks.size() != tasks.size()) {
        throw new RuntimeException("Can only do an identity grouping when source and target have same number of tasks");
    }
    tasks = new ArrayList<Integer>(tasks);
    Collections.sort(tasks);
    for (int i = 0; i < sourceTasks.size(); i++) {
        int s = sourceTasks.get(i);
        int t = tasks.get(i);
        _precomputed.put(s, Arrays.asList(t));
    }
}
 

/**
 * For bolts that has incoming streams from spouts (the root bolts),
 * add checkpoint stream from checkpoint spout to its input. For other bolts,
 * add checkpoint stream from the previous bolt to its input.
 */
private void addCheckPointInputs(ComponentCommon component) {
    Set<GlobalStreamId> checkPointInputs = new HashSet<>();
    for (GlobalStreamId inputStream : component.get_inputs().keySet()) {
        String sourceId = inputStream.get_componentId();
        if (_spouts.containsKey(sourceId)) {
            checkPointInputs.add(new GlobalStreamId(CheckpointSpout.CHECKPOINT_COMPONENT_ID, CheckpointSpout.CHECKPOINT_STREAM_ID));
        } else {
            checkPointInputs.add(new GlobalStreamId(sourceId, CheckpointSpout.CHECKPOINT_STREAM_ID));
        }
    }
    for (GlobalStreamId streamId : checkPointInputs) {
        component.put_to_inputs(streamId, Grouping.all(new NullStruct()));
    }
}
 

/**
 * Add watermark stream to source components of window bolts
 */
private void maybeAddWatermarkInputs(ComponentCommon common, IRichBolt bolt) {
    if (bolt instanceof WindowedBoltExecutor) {
        Set<String> comps = new HashSet<>();
        for (GlobalStreamId globalStreamId : common.get_inputs().keySet()) {
            comps.add(globalStreamId.get_componentId());
        }

        for (String comp : comps) {
            common.put_to_inputs(
                    new GlobalStreamId(comp, Common.WATERMARK_STREAM_ID),
                    Grouping.all(new NullStruct()));
        }
    }
}
 

public Map<String, List<Integer>> getThisSourceComponentTasks() {
    Map<String, List<Integer>> ret = new HashMap<>();
    Map<GlobalStreamId, Grouping> sources = getThisSources();
    Set<String> sourceComponents = new HashSet<>();
    if (sources != null) {
        for (GlobalStreamId globalStreamId : sources.keySet()) {
            sourceComponents.add(globalStreamId.get_componentId());
        }
    }
    for (String component : sourceComponents) {
        ret.put(component, getComponentTasks(component));
    }
    return ret;
}
 

Map<String, Set<String>> getBoltBatchToComponentSubscriptions(String id) {
    Map<String, Set<String>> ret = new HashMap();
    for(GlobalStreamId s: getBoltSubscriptionStreams(id)) {
        String b = _batchIds.get(s);
        if(!ret.containsKey(b)) ret.put(b, new HashSet());
        ret.get(b).add(s.get_componentId());
    }
    return ret;
}
 

List<GlobalStreamId> getBoltSubscriptionStreams(String id) {
    List<GlobalStreamId> ret = new ArrayList();
    Component c = _bolts.get(id);
    for(InputDeclaration d: c.declarations) {
        ret.add(new GlobalStreamId(d.getComponent(), d.getStream()));
    }
    return ret;
}
 

/**
 * Computes the min ts across all streams.
 */
private long computeWaterMarkTs() {
    long ts = 0;
    // only if some data has arrived on each input stream
    if (streamToTs.size() >= inputStreams.size()) {
        ts = Long.MAX_VALUE;
        for (Map.Entry<GlobalStreamId, Long> entry : streamToTs.entrySet()) {
            ts = Math.min(ts, entry.getValue());
        }
    }
    return ts - eventTsLag;
}
 

/**
 * Tracks the timestamp of the event in the stream, returns
 * true if the event can be considered for processing or
 * false if its a late event.
 */
public boolean track(GlobalStreamId stream, long ts) {
    Long currentVal = streamToTs.get(stream);
    if (currentVal == null || ts > currentVal) {
        streamToTs.put(stream, ts);
    }
    checkFailures();
    return ts >= lastWaterMarkTs;
}
 

@Override
public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
    this.targetTasks = targetTasks;
    targetTaskStats = new long[this.targetTasks.size()];
    if (this.fields != null) {
        this.outFields = context.getComponentOutputFields(stream);
    }
}
 

public static Set<String> getUpstreamComponents(String componentId, TopologyContext context) {
    Set<String> upstreamComponents = new HashSet<>();
    ComponentCommon componentCommon = Utils.getComponentCommon(context.getRawTopology(), componentId);
    Set<GlobalStreamId> input = componentCommon.get_inputs().keySet();
    for (GlobalStreamId stream : input) {
        upstreamComponents.add(stream.get_componentId());
    }
    return upstreamComponents;
}
 

@Override
public void prepare(org.apache.heron.api.topology.TopologyContext context,
                    String component, String streamId,
                    List<Integer> targetTasks) {
  TopologyContext c = new TopologyContext(context);
  GlobalStreamId g = new GlobalStreamId(component, streamId);
  delegate.prepare(c, g, targetTasks);
}
 

public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
  this.targetTasks = targetTasks;
  int numTasks = targetTasks.size();
  if (numTasks % numShards != 0)
    throw new IllegalArgumentException("Number of tasks ("+numTasks+") should be a multiple of the number of shards ("+numShards+")!");

  this.tasksPerShard = numTasks/numShards;
  this.random = new UniformIntegerDistribution(0, tasksPerShard-1);

  CompositeIdRouter docRouter =  new CompositeIdRouter();
  this.ranges = docRouter.partitionRange(numShards, docRouter.fullRange());
}
 

public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
  this.targetTasks = targetTasks;
  int numTasks = targetTasks.size();
  int numShards = initShardInfo(); // setup for doing shard to task mapping 
  if (numTasks % numShards != 0)
    throw new IllegalArgumentException("Number of tasks ("+numTasks+") should be a multiple of the number of shards ("+numShards+")!");

  this.tasksPerShard = numTasks/numShards;
  this.random = new UniformIntegerDistribution(0, tasksPerShard-1);
}
 

@Override
public void execute(Tuple tuple) {
    List<Object> id = tuple.select(_idFields);
    GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId());
    if (!_pending.containsKey(id)) {
        _pending.put(id, new HashMap<GlobalStreamId, Tuple>());
    }
    Map<GlobalStreamId, Tuple> parts = _pending.get(id);
    if (parts.containsKey(streamId))
        throw new RuntimeException("Received same side of single join twice");
    parts.put(streamId, tuple);
    if (parts.size() == _numSources) {
        _pending.remove(id);
        List<Object> joinResult = new ArrayList<Object>();
        for (String outField : _outFields) {
            GlobalStreamId loc = _fieldLocations.get(outField);
            joinResult.add(parts.get(loc).getValueByField(outField));
        }
        _collector.emit(new ArrayList<Tuple>(parts.values()), joinResult);

        for (Tuple part : parts.values()) {
            _collector.ack(part);
        }
        
        SingleJoinTest.receiveCounter.incrementAndGet();
    }
}
 

@Override
public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
    random = new Random();
    choices = new ArrayList<List<Integer>>(targetTasks.size());
    for (Integer i : targetTasks) {
        choices.add(Arrays.asList(i));
    }
    Collections.shuffle(choices, random);
    current = new AtomicInteger(0);
}
 

@Override
public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
    this.outdegreeTasks = new ArrayList<>(targetTasks);
    shuffleGroupingDelegate = new ShuffleGrouping();
    shuffleGroupingDelegate.prepare(context, stream, targetTasks);
    globalGroupingDelegate = new GlobalGrouping();
    globalGroupingDelegate.prepare(context, stream, targetTasks);
    connectedTargetIds = new HashMap<>();
    for (Integer targetId : targetTasks) {
        String targetComponentId = context.getComponentId(targetId);
        connectedTargetIds.put(targetComponentId, targetId);
    }
    LOG.info("OutDegree components: [{}]", StringUtils.join(connectedTargetIds.values(), ","));
}