backtype.storm.topology.IRichBolt Java Examples

@Test
public void testAddToTopology() {
  new Expectations() {
    {
      topology.getStormBuilder();
      result = stormBuilder;

      stormBuilder.setBolt(ID, (IRichBolt) any, anyInt);
      result = new MockUp<BoltDeclarer>() {
      }.getMockInstance();
    }
  };

  pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored

  new Verifications() {
    {
      assertEquals(pi.getProcessor(), processor);
      // TODO add methods to explore a topology and verify them
      assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2);
      assertEquals(pi.getId(), ID);
    }
  };
}
 

@Test
public void testAddToTopology() {
    new Expectations() {
        {
            topology.getStormBuilder();
            result = stormBuilder;

            stormBuilder.setBolt(ID, (IRichBolt) any, anyInt);
            result = new MockUp<BoltDeclarer>() {
            }.getMockInstance();
        }
    };

    pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored

    new Verifications() {
        {
            assertEquals(pi.getProcessor(), processor);
            // TODO add methods to explore a topology and verify them
            assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2);
            assertEquals(pi.getId(), ID);
        }
    };
}
 

@Override
public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelismHint) throws IllegalArgumentException{
    upToDownstreamComponentsMap.put(id, new HashSet<String>());
    validateUnusedId(id);
    IRichBolt boltExecutor;
    boolean isStatefulBolt = false;
    if (bolt instanceof ITransactionStatefulBoltExecutor) {
        isStatefulBolt = true;
        boltExecutor = new TransactionStatefulBolt((ITransactionStatefulBoltExecutor) bolt);
    } else {
        boltExecutor = new TransactionBolt((ITransactionBoltExecutor) bolt);
    }
    initCommon(id, boltExecutor, parallelismHint);
    _bolts.put(id, boltExecutor);
    BoltDeclarer ret = new TransactionBoltDeclarer(id);
    ret.addConfiguration(TransactionCommon.TRANSACTION_STATEFUL_BOLT, isStatefulBolt);

    // If using KvState bolt, the corresponding init operater would be registered here.
    if (bolt instanceof KvStatefulBoltExecutor) {
        ConfigExtension.registerTransactionTaskStateInitOp(TopologyBuilder.getStormConf(), id, KeyRangeStateTaskInit.class);
    }

    return ret;
}
 

private BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelism) {
    Integer p = null;
    if (parallelism != null)
        p = parallelism.intValue();
    Component component = new Component(bolt, p);
    _bolts.put(id, component);
    return new BoltDeclarerImpl(component);
}
 

/**
 * 根据Streaming算子创建Storm Bolt实例
 *
 */
public static IRichBolt createBolt(IRichOperator operator, StormConf stormConf) throws StreamingException
{
    if (operator instanceof OutputOperator)
    {
        return createOutputBolt((OutputOperator)operator, stormConf);
    }

    return CommonBoltCreator.createFunctionBolt(operator);
}
 

private BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelism, boolean committer) {
    Integer p = null;
    if (parallelism != null)
        p = parallelism.intValue();
    Component component = new Component(bolt, p, committer);
    _bolts.put(id, component);
    return new BoltDeclarerImpl(component);
}
 

public CoordinatedBolt(IRichBolt delegate, Map<String, SourceArgs> sourceArgs, IdStreamSpec idStreamSpec) {
    _sourceArgs = sourceArgs;
    if (_sourceArgs == null)
        _sourceArgs = new HashMap<>();
    _delegate = delegate;
    _idStreamSpec = idStreamSpec;
}
 

/**
 * 创建输出算子
 */
public static IRichBolt createOutputBolt(OutputOperator operator)
{
    StormOutputBolt outputBolt = new StormOutputBolt();
    outputBolt.setOperator(operator);
    return outputBolt;
}
 

/**
 * @return A topology builder than can further be customized.
 */
public TopologyBuilder create() {

    TopologyBuilder builder = new TopologyBuilder();

    if(eventsComponent instanceof IRichSpout)
      builder.setSpout(EVENT, (IRichSpout) eventsComponent, eventLoaderParallelism == -1 ? parallelismHint : eventLoaderParallelism);
    else if(eventsComponent instanceof IRichBolt)
      builder.setBolt(EVENT, (IRichBolt) eventsComponent, eventLoaderParallelism == -1 ? parallelismHint : eventLoaderParallelism);
    else
      throw new RuntimeException("The component for events is not valid. Must be IRichSpout or IRichBolt");


    if(flowLoaderSpout instanceof IRichSpout)
      builder.setSpout(FLOW_LOADER_STREAM, (IRichSpout) flowLoaderSpout, 1);
    else if(flowLoaderSpout instanceof IRichBolt)
      builder.setBolt(FLOW_LOADER_STREAM, (IRichBolt) flowLoaderSpout, 1);
    else
      throw new RuntimeException("The component for rules is not valid. Must be IRichSpout or IRichBolt");

    builder.setSpout("tick", new TickSpout(1000), 1);
    builder.setBolt(INITIALIZER, new FlowInitializerBolt(), parallelismHint)  // kicks off a flow determining where to start
            .localOrShuffleGrouping(EVENT)
            .allGrouping(FLOW_LOADER_STREAM, FLOW_LOADER_STREAM);

    declarebolt(builder, FILTER, new FilterBolt(), parallelismHint, true);
    declarebolt(builder, SELECT, new SelectorBolt(), parallelismHint, true);
    declarebolt(builder, PARTITION, new PartitionBolt(), parallelismHint, true);
    declarebolt(builder, SWITCH, new SwitchBolt(), parallelismHint, true);
    declarebolt(builder, AGGREGATE, new AggregatorBolt(), parallelismHint, true);
    declarebolt(builder, JOIN, new JoinBolt(), parallelismHint, true);
    declarebolt(builder, EACH, new EachBolt(), parallelismHint, true);
    declarebolt(builder, SORT, new SortBolt(), parallelismHint, true);
    declarebolt(builder, SPLIT, new SplitBolt(), parallelismHint, true);
    declarebolt(builder, OUTPUT, outputBolt, parallelismHint, false);

    return builder;
}
 

private static void declarebolt(TopologyBuilder builder, String boltName, IRichBolt bolt, int parallelism, boolean control) {
    BoltDeclarer declarer = builder.setBolt(boltName, bolt, parallelism)
        .allGrouping(FLOW_LOADER_STREAM, FLOW_LOADER_STREAM)
        .allGrouping("tick", "tick")
        .localOrShuffleGrouping(INITIALIZER, boltName)
        .localOrShuffleGrouping(FILTER, boltName)
        .fieldsGrouping(PARTITION, boltName, new Fields(FLOW_ID, PARTITION))    // guaranteed partitions will always group the same flow for flows that have joins with default partitions.
        .localOrShuffleGrouping(AGGREGATE, boltName)
        .localOrShuffleGrouping(SELECT, boltName)
        .localOrShuffleGrouping(EACH, boltName)
        .localOrShuffleGrouping(SORT, boltName)
        .localOrShuffleGrouping(SWITCH, boltName)
        .localOrShuffleGrouping(SPLIT, boltName)
        .localOrShuffleGrouping(JOIN, boltName);


        if(control) {
          // control stream is all-grouped
          declarer.allGrouping(INITIALIZER, BROADCAST_STREAM + boltName)
                  .allGrouping(FILTER, BROADCAST_STREAM + boltName)
                  .allGrouping(PARTITION, BROADCAST_STREAM + boltName)
                  .allGrouping(AGGREGATE, BROADCAST_STREAM + boltName)
                  .allGrouping(SELECT, BROADCAST_STREAM + boltName)
                  .allGrouping(EACH, BROADCAST_STREAM + boltName)
                  .allGrouping(SORT, BROADCAST_STREAM + boltName)
                  .allGrouping(SWITCH, BROADCAST_STREAM + boltName)
                  .allGrouping(SPLIT, BROADCAST_STREAM + boltName)
                  .allGrouping(JOIN, BROADCAST_STREAM + boltName);

        }
}
 

@Deprecated
public LinearDRPCInputDeclarer addBolt(IRichBolt bolt, Number parallelism) {
    if (parallelism == null)
        parallelism = 1;
    Component component = new Component(bolt, parallelism.intValue());
    _components.add(component);
    return new InputDeclarerImpl(component);
}
 

private BoltDeclarer createBoltDeclarer(IRichOperator operator) {
	IRichBolt bolt;
	if ((operator instanceof FunctionOperator) || (operator instanceof FunctionStreamOperator)) {
		bolt = createStormBolt(operator);
	} else {
		bolt = createOutputStormBolt(operator);
	}
	return builder.setBolt(operator.getOperatorId(), bolt, operator.getParallelNumber());
}
 

public KeyedFairBolt(IRichBolt delegate) {
    _delegate = delegate;
}
 

public AckTransactionBolt(IRichBolt bolt) {
    this.bolt = bolt;
}
 

public Component(IRichBolt bolt, Integer parallelism) {
    this.bolt = bolt;
    this.parallelism = parallelism;
}
 

public CoordinatedBolt(IRichBolt delegate) {
    this(delegate, null, null);
}
 

public CoordinatedBolt(IRichBolt delegate, String sourceComponent, SourceArgs sourceArgs, IdStreamSpec idStreamSpec) {
    this(delegate, singleSourceArgs(sourceComponent, sourceArgs), idStreamSpec);
}
 

public Component(IRichBolt bolt, Integer parallelism, boolean committer) {
    this.bolt = bolt;
    this.parallelism = parallelism;
    this.committer = committer;
}
 

public BoltTracker(IRichBolt delegate, String id) {
    super(delegate, id);
    _richDelegate = delegate;
}
 

public Component(IRichBolt bolt, int parallelism) {
    this.bolt = bolt;
    this.parallelism = parallelism;
    this.componentConfs = new ArrayList();
}
 

private StormTopology createTopology(DRPCSpout spout) {
    final String SPOUT_ID = "spout";
    final String PREPARE_ID = "prepare-request";

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout);
    builder.setBolt(PREPARE_ID, new PrepareRequest()).noneGrouping(SPOUT_ID);
    int i = 0;
    for (; i < _components.size(); i++) {
        Component component = _components.get(i);

        Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
        if (i == 1) {
            source.put(boltId(i - 1), SourceArgs.single());
        } else if (i >= 2) {
            source.put(boltId(i - 1), SourceArgs.all());
        }
        IdStreamSpec idSpec = null;
        if (i == _components.size() - 1 && component.bolt instanceof FinishedCallback) {
            idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
        }
        BoltDeclarer declarer = builder.setBolt(boltId(i), new CoordinatedBolt(component.bolt, source, idSpec), component.parallelism);

        for (Map conf : component.componentConfs) {
            declarer.addConfigurations(conf);
        }

        if (idSpec != null) {
            declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
        }
        if (i == 0 && component.declarations.isEmpty()) {
            declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
        } else {
            String prevId;
            if (i == 0) {
                prevId = PREPARE_ID;
            } else {
                prevId = boltId(i - 1);
            }
            for (InputDeclaration declaration : component.declarations) {
                declaration.declare(prevId, declarer);
            }
        }
        if (i > 0) {
            declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID);
        }
    }

    IRichBolt lastBolt = _components.get(_components.size() - 1).bolt;
    OutputFieldsGetter getter = new OutputFieldsGetter();
    lastBolt.declareOutputFields(getter);
    Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
    if (streams.size() != 1) {
        throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
    }
    String outputStream = streams.keySet().iterator().next();
    List<String> fields = streams.get(outputStream).get_output_fields();
    if (fields.size() != 2) {
        throw new RuntimeException(
                "Output stream of last component in LinearDRPCTopology must contain exactly two fields. The first should be the request id, and the second should be the result.");
    }

    builder.setBolt("JoinResult", new JoinResult(PREPARE_ID)).fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0)))
            .fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
    i++;
    builder.setBolt("ReturnResults", new ReturnResults()).noneGrouping("JoinResult");
    return builder.createTopology();
}
 

@Deprecated
public LinearDRPCInputDeclarer addBolt(IRichBolt bolt) {
    return addBolt(bolt, null);
}
 

public FlowmixBuilder setOutputBolt(IRichBolt outputBolt) {
  this.outputBolt = outputBolt;
  return this;
}
 

/**
 * 创建功能性算子
 */
public static IRichBolt createFunctionBolt(IRichOperator operator){
    StormBolt bolt = new StormBolt();
    bolt.setOperator(operator);
    return bolt;
}
 

private static IRichBolt createOutputBolt(OutputOperator operator, StormConf stormConf) throws StreamingException
{
    return CommonBoltCreator.createOutputBolt(operator);
}
 

private IRichBolt createStormBolt(IRichOperator f) {
	StormBolt stormbolt = new StormBolt();
	stormbolt.setOperator(f);
	return stormbolt;
}
 

private IRichBolt createOutputStormBolt(IRichOperator f) {
	StormOutputBolt outputbolt = new StormOutputBolt();
	outputbolt.setOperator(f);
	return outputbolt;
}
 

/**
 * Build bolt to provide the compatibility with Storm's ack mechanism
 *
 * @param id bolt Id
 * @param bolt
 * @return
 */
public BoltDeclarer setBoltWithAck(String id, IRichBolt bolt, Number parallelismHint) {
    return setBolt(id, new AckTransactionBolt(bolt), parallelismHint);
}