backtype.storm.topology.BoltDeclarer Java Examples

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

@Test
public void testAddToTopology() {
    new Expectations() {
        {
            topology.getStormBuilder();
            result = stormBuilder;

            stormBuilder.setBolt(ID, (IRichBolt) any, anyInt);
            result = new MockUp<BoltDeclarer>() {
            }.getMockInstance();
        }
    };

    pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored

    new Verifications() {
        {
            assertEquals(pi.getProcessor(), processor);
            // TODO add methods to explore a topology and verify them
            assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2);
            assertEquals(pi.getId(), ID);
        }
    };
}
 

@Override
public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelismHint) throws IllegalArgumentException{
    upToDownstreamComponentsMap.put(id, new HashSet<String>());
    validateUnusedId(id);
    IRichBolt boltExecutor;
    boolean isStatefulBolt = false;
    if (bolt instanceof ITransactionStatefulBoltExecutor) {
        isStatefulBolt = true;
        boltExecutor = new TransactionStatefulBolt((ITransactionStatefulBoltExecutor) bolt);
    } else {
        boltExecutor = new TransactionBolt((ITransactionBoltExecutor) bolt);
    }
    initCommon(id, boltExecutor, parallelismHint);
    _bolts.put(id, boltExecutor);
    BoltDeclarer ret = new TransactionBoltDeclarer(id);
    ret.addConfiguration(TransactionCommon.TRANSACTION_STATEFUL_BOLT, isStatefulBolt);

    // If using KvState bolt, the corresponding init operater would be registered here.
    if (bolt instanceof KvStatefulBoltExecutor) {
        ConfigExtension.registerTransactionTaskStateInitOp(TopologyBuilder.getStormConf(), id, KeyRangeStateTaskInit.class);
    }

    return ret;
}
 

@Test
public void testAddToTopology() {
  new Expectations() {
    {
      topology.getStormBuilder();
      result = stormBuilder;

      stormBuilder.setBolt(ID, (IRichBolt) any, anyInt);
      result = new MockUp<BoltDeclarer>() {
      }.getMockInstance();
    }
  };

  pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored

  new Verifications() {
    {
      assertEquals(pi.getProcessor(), processor);
      // TODO add methods to explore a topology and verify them
      assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2);
      assertEquals(pi.getId(), ID);
    }
  };
}
 

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

public BoltDeclarer setBolt(String id, IBasicBolt bolt, int parallel) {
    CoordinatedBolt coordinatedBolt = new CoordinatedBolt(bolt);

    BoltDeclarer boltDeclarer = topologyBuilder.setBolt(id, coordinatedBolt, parallel);

    if (bolt instanceof IPrepareCommit) {
        boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.PREPARE_STREAM_ID);
    }

    if (bolt instanceof ICommitter) {
        boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.COMMIT_STREAM_ID);
        boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.REVERT_STREAM_ID);
    }

    if (bolt instanceof IPostCommit) {
        boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.POST_STREAM_ID);
    }

    return boltDeclarer;
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfAnalyzerTasks = config.getInt(ANALYZER_TASK_NUM);
    int numOfGeneratorTasks = config.getInt(GENERATOR_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);

    GCLogAnalyzerBolt bolt = new GCLogAnalyzerBolt();
    BoltDeclarer boltDeclarer = builder.setBolt("analyzerBolt", bolt, numOfAnalyzerTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    GCMetricGeneratorBolt generatorBolt = new GCMetricGeneratorBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("generatorBolt", generatorBolt, numOfGeneratorTasks);
    joinBoltDeclarer.fieldsGrouping("analyzerBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("gc_log_stream",config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("generatorBolt", new Fields("f1"));
    return builder.createTopology();
}
 

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

public static void main(String[] args) throws Exception{
    System.setProperty("config.resource", "/application.conf");
    Config config = ConfigFactory.load();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    SecurityLogParserBolt bolt = new SecurityLogParserBolt();
    TopologyBuilder builder = new TopologyBuilder();

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
    boltDeclarer.shuffleGrouping("ingest");

    KafkaBolt kafkaBolt = new KafkaBolt();
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", kafkaBolt, numOfSinkTasks);
    kafkaBoltDeclarer.shuffleGrouping("parserBolt");

    StormTopology topology = builder.createTopology();

    TopologySubmitter.submit(topology, config);
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    String deserClsName = config.getString("dataSourceConfig.deserializerClass");
    KafkaSourcedSpoutScheme scheme = new KafkaSourcedSpoutScheme(deserClsName, config);

    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout spout1 = new KafkaOffsetSourceSpoutProvider().getSpout(config);
    BaseRichSpout spout2 = KafkaSourcedSpoutProvider.getSpout(config, scheme);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfDistributionTasks = config.getInt(DISTRIBUTION_TASK_NUM);

    builder.setSpout("kafkaLogLagChecker", spout1, numOfSpoutTasks);
    builder.setSpout("kafkaMessageFetcher", spout2, numOfSpoutTasks);

    KafkaMessageDistributionBolt bolt = new KafkaMessageDistributionBolt(config);
    BoltDeclarer bolteclarer = builder.setBolt("distributionBolt", bolt, numOfDistributionTasks);
    bolteclarer.fieldsGrouping("kafkaLogLagChecker", new Fields("f1"));
    bolteclarer.fieldsGrouping("kafkaLogLagChecker", new Fields("f1"));
    return builder.createTopology();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    HBaseAuditLogParserBolt bolt = new HBaseAuditLogParserBolt();

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    HBaseResourceSensitivityDataJoinBolt joinBolt = new HBaseResourceSensitivityDataJoinBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("hbase_audit_log_stream",config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}
 

@Override
public BoltDeclarer noneGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.noneGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer directGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.directGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer localFirstGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localFirstGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer shuffleGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.shuffleGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer localOrShuffleGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localOrShuffleGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer localOrShuffleGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localOrShuffleGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer fieldsGrouping(final String component, final String streamId, final Fields fields) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.fieldsGrouping(component, streamId, fields);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer allGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.allGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer allGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.allGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer noneGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.noneGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer noneGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.noneGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer localFirstGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localFirstGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer localFirstGrouping(final String componentId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localFirstGrouping(componentId);
        }

        @Override
        public String getComponent() {
            return componentId;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer localOrShuffleGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localOrShuffleGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer shuffleGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.shuffleGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer shuffleGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.shuffleGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer globalGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.globalGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer globalGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.globalGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}
 

@Override
public BoltDeclarer fieldsGrouping(final String component, final Fields fields) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.fieldsGrouping(component, fields);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}