Java Code Examples for backtype.storm.topology.BoltDeclarer#fieldsGrouping()
The following examples show how to use
backtype.storm.topology.BoltDeclarer#fieldsGrouping() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HBaseAuditLogApplication.java From eagle with Apache License 2.0 | 6 votes |
|
@Override
public StormTopology execute(Config config, StormEnvironment environment) {
TopologyBuilder builder = new TopologyBuilder();
KafkaSpoutProvider provider = new KafkaSpoutProvider();
IRichSpout spout = provider.getSpout(config);
HBaseAuditLogParserBolt bolt = new HBaseAuditLogParserBolt();
int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
builder.setSpout("ingest", spout, numOfSpoutTasks);
BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));
HBaseResourceSensitivityDataJoinBolt joinBolt = new HBaseResourceSensitivityDataJoinBolt(config);
BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
StormStreamSink sinkBolt = environment.getStreamSink("hbase_audit_log_stream",config);
BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
return builder.createTopology();
}
Example 2
| Source File: OozieAuditLogApplication.java From eagle with Apache License 2.0 | 6 votes |
|
@Override
public StormTopology execute(Config config, StormEnvironment environment) {
TopologyBuilder builder = new TopologyBuilder();
KafkaSpoutProvider provider = new KafkaSpoutProvider();
IRichSpout spout = provider.getSpout(config);
int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
int numOfParserTask = config.getInt(PARSER_TASK_NUM);
int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
builder.setSpout("ingest", spout, numOfSpoutTasks);
OozieAuditLogParserBolt parserBolt = new OozieAuditLogParserBolt();
BoltDeclarer parserBoltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTask);
parserBoltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));
OozieResourceSensitivityDataJoinBolt joinBolt = new OozieResourceSensitivityDataJoinBolt(config);
BoltDeclarer boltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
boltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
StormStreamSink sinkBolt = environment.getStreamSink("oozie_audit_log_stream", config);
int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
return builder.createTopology();
}
Example 3
| Source File: EagleMetricCollectorApplication.java From eagle with Apache License 2.0 | 6 votes |
|
@Override
public StormTopology execute(Config config, StormEnvironment environment) {
String deserClsName = config.getString("dataSourceConfig.deserializerClass");
KafkaSourcedSpoutScheme scheme = new KafkaSourcedSpoutScheme(deserClsName, config);
TopologyBuilder builder = new TopologyBuilder();
BaseRichSpout spout1 = new KafkaOffsetSourceSpoutProvider().getSpout(config);
BaseRichSpout spout2 = KafkaSourcedSpoutProvider.getSpout(config, scheme);
int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
int numOfDistributionTasks = config.getInt(DISTRIBUTION_TASK_NUM);
builder.setSpout("kafkaLogLagChecker", spout1, numOfSpoutTasks);
builder.setSpout("kafkaMessageFetcher", spout2, numOfSpoutTasks);
KafkaMessageDistributionBolt bolt = new KafkaMessageDistributionBolt(config);
BoltDeclarer bolteclarer = builder.setBolt("distributionBolt", bolt, numOfDistributionTasks);
bolteclarer.fieldsGrouping("kafkaLogLagChecker", new Fields("f1"));
bolteclarer.fieldsGrouping("kafkaLogLagChecker", new Fields("f1"));
return builder.createTopology();
}
Example 4
| Source File: GCLogApplication.java From eagle with Apache License 2.0 | 6 votes |
|
@Override
public StormTopology execute(Config config, StormEnvironment environment) {
TopologyBuilder builder = new TopologyBuilder();
KafkaSpoutProvider provider = new KafkaSpoutProvider();
IRichSpout spout = provider.getSpout(config);
int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
int numOfAnalyzerTasks = config.getInt(ANALYZER_TASK_NUM);
int numOfGeneratorTasks = config.getInt(GENERATOR_TASK_NUM);
int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
builder.setSpout("ingest", spout, numOfSpoutTasks);
GCLogAnalyzerBolt bolt = new GCLogAnalyzerBolt();
BoltDeclarer boltDeclarer = builder.setBolt("analyzerBolt", bolt, numOfAnalyzerTasks);
boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));
GCMetricGeneratorBolt generatorBolt = new GCMetricGeneratorBolt(config);
BoltDeclarer joinBoltDeclarer = builder.setBolt("generatorBolt", generatorBolt, numOfGeneratorTasks);
joinBoltDeclarer.fieldsGrouping("analyzerBolt", new Fields("f1"));
StormStreamSink sinkBolt = environment.getStreamSink("gc_log_stream",config);
BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
kafkaBoltDeclarer.fieldsGrouping("generatorBolt", new Fields("f1"));
return builder.createTopology();
}
Example 5
| Source File: JStormApplication.java From PoseidonX with Apache License 2.0 | 5 votes |
|
private void setBoltGrouping(BoltDeclarer bolt, String strname, GroupInfo groupInfo) throws StreamingException {
if (null == groupInfo) {
setDefaultBoltGrouping(bolt, strname);
return;
}
DistributeType distribute = groupInfo.getDitributeType();
switch (distribute) {
case FIELDS:
Fields fields = new Fields(groupInfo.getFields());
IRichOperator operator = getOperatorByOutputStreamName(strname);
if (operator == null) {
StreamingException exception = new StreamingException(ErrorCode.PLATFORM_INVALID_TOPOLOGY);
LOG.error("Can't find opertor by stream name : {} .", strname, exception);
throw exception;
}
bolt.fieldsGrouping(operator.getOperatorId(), strname, fields);
break;
case GLOBAL:
break;
case LOCALORSHUFFLE:
break;
case ALL:
break;
case DIRECT:
break;
case CUSTOM:
break;
case SHUFFLE:
case NONE:
default:
setDefaultBoltGrouping(bolt, strname);
}
}
Example 6
| Source File: HiveQueryMonitoringApplication.java From eagle with Apache License 2.0 | 5 votes |
|
@Override
public StormTopology execute(Config config, StormEnvironment environment) {
TopologyBuilder builder = new TopologyBuilder();
HiveJobRunningSourcedStormSpoutProvider provider = new HiveJobRunningSourcedStormSpoutProvider();
IRichSpout spout = provider.getSpout(config, config.getInt(SPOUT_TASK_NUM));
int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
int numOfFilterTasks = config.getInt(FILTER_TASK_NUM);
int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
builder.setSpout("ingest", spout, numOfSpoutTasks);
JobFilterBolt bolt = new JobFilterBolt();
BoltDeclarer boltDeclarer = builder.setBolt("filterBolt", bolt, numOfFilterTasks);
boltDeclarer.fieldsGrouping("ingest", new Fields("jobId"));
HiveQueryParserBolt parserBolt = new HiveQueryParserBolt();
BoltDeclarer parserBoltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTasks);
parserBoltDeclarer.fieldsGrouping("filterBolt", new Fields("user"));
HiveSensitivityDataEnrichBolt joinBolt = new HiveSensitivityDataEnrichBolt(config);
BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("user"));
StormStreamSink sinkBolt = environment.getStreamSink("hive_query_stream", config);
BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
return builder.createTopology();
}
Example 7
| Source File: AlertTopologyTest.java From eagle with Apache License 2.0 | 4 votes |
|
@Ignore
@Test
public void testMultipleTopics() throws Exception {
final String topoId = "myTopology";
int numGroupbyBolts = 2;
int numTotalGroupbyBolts = 3;
System.setProperty("eagle.correlation.numGroupbyBolts", String.valueOf(numGroupbyBolts));
System.setProperty("eagle.correlation.topologyName", topoId);
System.setProperty("eagle.correlation.mode", "local");
System.setProperty("eagle.correlation.zkHosts", "localhost:2181");
final String topicName1 = "testTopic3";
final String topicName2 = "testTopic4";
// ensure topic ready
LogManager.getLogger(CorrelationSpout.class).setLevel(Level.DEBUG);
Config config = ConfigFactory.load();
CreateTopicUtils.ensureTopicReady(System.getProperty("eagle.correlation.zkHosts"), topicName1);
CreateTopicUtils.ensureTopicReady(System.getProperty("eagle.correlation.zkHosts"), topicName2);
TopologyBuilder topoBuilder = new TopologyBuilder();
int numBolts = config.getInt("eagle.correlation.numGroupbyBolts");
CorrelationSpout spout = new CorrelationSpout(config, topoId, null, numBolts);
String spoutId = "correlation-spout";
SpoutDeclarer declarer = topoBuilder.setSpout(spoutId, spout);
for (int i = 0; i < numBolts; i++) {
TestBolt bolt = new TestBolt();
BoltDeclarer boltDecl = topoBuilder.setBolt("engineBolt" + i, bolt);
boltDecl.fieldsGrouping(spoutId, "stream_" + i, new Fields());
}
String topoName = config.getString("eagle.correlation.topologyName");
LOG.info("start topology in local mode");
LocalCluster cluster = new LocalCluster();
cluster.submitTopology(topoName, new HashMap<>(), topoBuilder.createTopology());
while (true) {
try {
Thread.sleep(1000);
} catch (Exception e) {
e.printStackTrace();
}
}
}
Example 8
| Source File: CoordinatorSpoutIntegrationTest.java From eagle with Apache License 2.0 | 4 votes |
|
@Ignore // this test need zookeeper
@Test
public void testConfigNotify() throws Exception {
final String topoId = "myTopology";
int numGroupbyBolts = 2;
int numTotalGroupbyBolts = 3;
System.setProperty("correlation.serviceHost", "sandbox.hortonworks.com");
System.setProperty("correlation.servicePort", "58080");
System.setProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum", "sandbox.hortonworks.com:2181");
System.setProperty("correlation.numGroupbyBolts", String.valueOf(numGroupbyBolts));
System.setProperty("correlation.topologyName", topoId);
System.setProperty("correlation.mode", "local");
System.setProperty("correlation.zkHosts", "sandbox.hortonworks.com:2181");
final String topicName1 = "testTopic3";
final String topicName2 = "testTopic4";
// ensure topic ready
LogManager.getLogger(CorrelationSpout.class).setLevel(Level.DEBUG);
Config config = ConfigFactory.load();
CreateTopicUtils.ensureTopicReady(System.getProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum"), topicName1);
CreateTopicUtils.ensureTopicReady(System.getProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum"), topicName2);
TopologyBuilder topoBuilder = new TopologyBuilder();
int numBolts = config.getInt("correlation.numGroupbyBolts");
String spoutId = "correlation-spout";
CorrelationSpout spout = new CorrelationSpout(config, topoId,
new MockMetadataChangeNotifyService(topoId, spoutId), numBolts);
SpoutDeclarer declarer = topoBuilder.setSpout(spoutId, spout);
declarer.setNumTasks(2);
for (int i = 0; i < numBolts; i++) {
TestBolt bolt = new TestBolt();
BoltDeclarer boltDecl = topoBuilder.setBolt("engineBolt" + i, bolt);
boltDecl.fieldsGrouping(spoutId,
StreamIdConversion.generateStreamIdBetween(AlertConstants.DEFAULT_SPOUT_NAME, AlertConstants.DEFAULT_ROUTERBOLT_NAME + i), new Fields());
}
String topoName = config.getString("correlation.topologyName");
LOG.info("start topology in local mode");
LocalCluster cluster = new LocalCluster();
StormTopology topology = topoBuilder.createTopology();
cluster.submitTopology(topoName, new HashMap(), topology);
Utils.sleep(Long.MAX_VALUE);
}
Example 9
| Source File: TopologyRunner.java From opensoc-streaming with Apache License 2.0 | 4 votes |
|
public boolean initializeHbaseBolt(String name, String shuffleType) {
try {
String messageUpstreamComponent = dataComponents.get(dataComponents
.size()-1);
System.out.println("[OpenSOC] ------" + name
+ " is initializing from " + messageUpstreamComponent);
String tableName = config.getString("bolt.hbase.table.name")
.toString();
TupleTableConfig hbaseBoltConfig = new TupleTableConfig(tableName,
config.getString("bolt.hbase.table.key.tuple.field.name")
.toString(), config.getString(
"bolt.hbase.table.timestamp.tuple.field.name")
.toString());
String allColumnFamiliesColumnQualifiers = config.getString(
"bolt.hbase.table.fields").toString();
// This is expected in the form
// "<cf1>:<cq11>,<cq12>,<cq13>|<cf2>:<cq21>,<cq22>|......."
String[] tokenizedColumnFamiliesWithColumnQualifiers = StringUtils
.split(allColumnFamiliesColumnQualifiers, "\\|");
for (String tokenizedColumnFamilyWithColumnQualifiers : tokenizedColumnFamiliesWithColumnQualifiers) {
String[] cfCqTokens = StringUtils.split(
tokenizedColumnFamilyWithColumnQualifiers, ":");
String columnFamily = cfCqTokens[0];
String[] columnQualifiers = StringUtils.split(cfCqTokens[1],
",");
for (String columnQualifier : columnQualifiers) {
hbaseBoltConfig.addColumn(columnFamily, columnQualifier);
}
// hbaseBoltConfig.setDurability(Durability.valueOf(conf.get(
// "storm.topology.pcap.bolt.hbase.durability").toString()));
hbaseBoltConfig.setBatch(Boolean.valueOf(config.getString(
"bolt.hbase.enable.batching").toString()));
HBaseBolt hbase_bolt = new HBaseBolt(hbaseBoltConfig,
config.getString("kafka.zk.list"),
config.getString("kafka.zk.port"));
hbase_bolt.setAutoAck(true);
BoltDeclarer declarer = builder.setBolt(name, hbase_bolt,
config.getInt("bolt.hbase.parallelism.hint"))
.setNumTasks(config.getInt("bolt.hbase.num.tasks"));
if (Grouping._Fields.CUSTOM_OBJECT.toString().equalsIgnoreCase(
shuffleType)) {
declarer.customGrouping(
messageUpstreamComponent,
"pcap_data_stream",
new HBaseStreamPartitioner(
hbaseBoltConfig.getTableName(),
0,
Integer.parseInt(conf
.get("bolt.hbase.partitioner.region.info.refresh.interval.mins")
.toString())));
} else if (Grouping._Fields.DIRECT.toString().equalsIgnoreCase(
shuffleType)) {
declarer.fieldsGrouping(messageUpstreamComponent,
"pcap_data_stream", new Fields("pcap_id"));
}
}
} catch (Exception e) {
e.printStackTrace();
System.exit(0);
}
return true;
}
Example 10
| Source File: LinearDRPCTopologyBuilder.java From jstorm with Apache License 2.0 | 4 votes |
|
private StormTopology createTopology(DRPCSpout spout) {
final String SPOUT_ID = "spout";
final String PREPARE_ID = "prepare-request";
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, spout);
builder.setBolt(PREPARE_ID, new PrepareRequest()).noneGrouping(SPOUT_ID);
int i = 0;
for (; i < _components.size(); i++) {
Component component = _components.get(i);
Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
if (i == 1) {
source.put(boltId(i - 1), SourceArgs.single());
} else if (i >= 2) {
source.put(boltId(i - 1), SourceArgs.all());
}
IdStreamSpec idSpec = null;
if (i == _components.size() - 1 && component.bolt instanceof FinishedCallback) {
idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
}
BoltDeclarer declarer = builder.setBolt(boltId(i), new CoordinatedBolt(component.bolt, source, idSpec), component.parallelism);
for (Map conf : component.componentConfs) {
declarer.addConfigurations(conf);
}
if (idSpec != null) {
declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
}
if (i == 0 && component.declarations.isEmpty()) {
declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
} else {
String prevId;
if (i == 0) {
prevId = PREPARE_ID;
} else {
prevId = boltId(i - 1);
}
for (InputDeclaration declaration : component.declarations) {
declaration.declare(prevId, declarer);
}
}
if (i > 0) {
declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID);
}
}
IRichBolt lastBolt = _components.get(_components.size() - 1).bolt;
OutputFieldsGetter getter = new OutputFieldsGetter();
lastBolt.declareOutputFields(getter);
Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
if (streams.size() != 1) {
throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
}
String outputStream = streams.keySet().iterator().next();
List<String> fields = streams.get(outputStream).get_output_fields();
if (fields.size() != 2) {
throw new RuntimeException(
"Output stream of last component in LinearDRPCTopology must contain exactly two fields. The first should be the request id, and the second should be the result.");
}
builder.setBolt("JoinResult", new JoinResult(PREPARE_ID)).fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0)))
.fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
i++;
builder.setBolt("ReturnResults", new ReturnResults()).noneGrouping("JoinResult");
return builder.createTopology();
}
