Java Code Examples for org.apache.storm.Config#setNumAckers()
The following examples show how to use
org.apache.storm.Config#setNumAckers() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ScottyDemoTopology.java From scotty-window-processor with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
LocalCluster cluster = new LocalCluster();
TopologyBuilder builder = new TopologyBuilder();
Config conf = new Config();
conf.setDebug(false);
conf.setNumWorkers(1);
conf.setMaxTaskParallelism(1);
//Disable Acking
conf.setNumAckers(0);
KeyedScottyWindowOperator scottyBolt = new KeyedScottyWindowOperator<Integer, Integer>(new Sum(), 0);
scottyBolt.addWindow(new TumblingWindow(WindowMeasure.Time, 1000));
scottyBolt.addWindow(new SlidingWindow(WindowMeasure.Time, 1000, 250));
scottyBolt.addWindow(new SessionWindow(WindowMeasure.Time, 1000));
builder.setSpout("spout", new DataGeneratorSpout());
builder.setBolt("scottyWindow", scottyBolt).fieldsGrouping("spout", new Fields("key"));
builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("scottyWindow");
cluster.submitTopology("testTopology", conf, builder.createTopology());
//cluster.killTopology("testTopology");
//cluster.shutdown();
}
Example 2
| Source File: AckingTopology.java From incubator-heron with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
if (args.length != 1) {
throw new RuntimeException("Specify topology name");
}
TopologyBuilder builder = new TopologyBuilder();
int spouts = 2;
int bolts = 2;
builder.setSpout("word", new AckingTestWordSpout(), spouts);
builder.setBolt("exclaim1", new ExclamationBolt(), bolts)
.shuffleGrouping("word");
Config conf = new Config();
conf.setDebug(true);
// Put an arbitrary large number here if you don't want to slow the topology down
conf.setMaxSpoutPending(1000 * 1000 * 1000);
// To enable acking, we need to setEnableAcking true
conf.setNumAckers(1);
conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
// Set the number of workers or stream managers
conf.setNumWorkers(2);
StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
Example 3
| Source File: TopologyApp.java From springBoot-study with Apache License 2.0 | 5 votes |
|
public void runStorm(String[] args) {
// 定义一个拓扑
TopologyBuilder builder = new TopologyBuilder();
// 设置1个Executeor(线程)，默认一个
builder.setSpout(Constants.KAFKA_SPOUT, new KafkaInsertDataSpout(), 1);
// shuffleGrouping:表示是随机分组
// 设置1个Executeor(线程)，和两个task
builder.setBolt(Constants.INSERT_BOLT, new InsertBolt(), 1).setNumTasks(1).shuffleGrouping(Constants.KAFKA_SPOUT);
Config conf = new Config();
//设置一个应答者
conf.setNumAckers(1);
//设置一个work
conf.setNumWorkers(3);
try {
// 有参数时，表示向集群提交作业，并把第一个参数当做topology名称
// 没有参数时，本地提交
if (args != null && args.length > 0) {
logger.info("运行远程模式");
StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
} else {
// 启动本地模式
logger.info("运行本地模式");
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("TopologyApp", conf, builder.createTopology());
}
} catch (Exception e) {
logger.error("storm启动失败!程序退出!",e);
System.exit(1);
}
logger.info("storm启动成功...");
}
Example 4
| Source File: PirkTopology.java From incubator-retired-pirk with Apache License 2.0 | 5 votes |
|
public static Config createStormConf()
{
Boolean limitHitsPerSelector = Boolean.parseBoolean(SystemConfiguration.getProperty("pir.limitHitsPerSelector"));
Integer maxHitsPerSelector = Integer.parseInt(SystemConfiguration.getProperty("pir.maxHitsPerSelector"));
Integer rowDivisions = Integer.parseInt(SystemConfiguration.getProperty("storm.rowDivs", "1"));
Config conf = new Config();
conf.setNumAckers(Integer.parseInt(SystemConfiguration.getProperty("storm.numAckers", numWorkers.toString())));
conf.setMaxSpoutPending(Integer.parseInt(SystemConfiguration.getProperty("storm.maxSpoutPending", "300")));
conf.setNumWorkers(numWorkers);
conf.setDebug(false);
// conf.setNumEventLoggers(2);
conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, SystemConfiguration.getIntProperty("storm.executor.receiveBufferSize", 1024));
conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, SystemConfiguration.getIntProperty("storm.executor.sendBufferSize", 1024));
conf.put(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, SystemConfiguration.getIntProperty("storm.transferBufferSize", 32));
conf.put(Config.WORKER_HEAP_MEMORY_MB, SystemConfiguration.getIntProperty("storm.worker.heapMemory", 750));
conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, Double.parseDouble(SystemConfiguration.getProperty("storm.componentOnheapMem", "128")));
// Pirk parameters to send to bolts
conf.put(StormConstants.ALLOW_ADHOC_QSCHEMAS_KEY, SystemConfiguration.getProperty("pir.allowAdHocQuerySchemas", "false").equals("true"));
conf.put(StormConstants.QSCHEMA_KEY, SystemConfiguration.getProperty("query.schemas"));
conf.put(StormConstants.DSCHEMA_KEY, SystemConfiguration.getProperty("data.schemas"));
conf.put(StormConstants.HDFS_URI_KEY, hdfsUri);
conf.put(StormConstants.QUERY_FILE_KEY, queryFile);
conf.put(StormConstants.USE_HDFS, useHdfs);
conf.put(StormConstants.OUTPUT_FILE_KEY, outputPath);
conf.put(StormConstants.LIMIT_HITS_PER_SEL_KEY, limitHitsPerSelector);
conf.put(StormConstants.MAX_HITS_PER_SEL_KEY, maxHitsPerSelector);
conf.put(StormConstants.SPLIT_PARTITIONS_KEY, splitPartitions);
conf.put(StormConstants.SALT_COLUMNS_KEY, saltColumns);
conf.put(StormConstants.ROW_DIVISIONS_KEY, rowDivisions);
conf.put(StormConstants.ENCROWCALCBOLT_PARALLELISM_KEY, encrowcalcboltParallelism);
conf.put(StormConstants.ENCCOLMULTBOLT_PARALLELISM_KEY, enccolmultboltParallelism);
return conf;
}
Example 5
| Source File: DispatcherAppenderTopology.java From DBus with Apache License 2.0 | 4 votes |
|
private void start(StormTopology topology, boolean runAsLocal) throws Exception {
Config conf = new Config();
// 启动类型为all，或者dispatcher
if (topologyType.equals(Constants.TopologyType.ALL) || topologyType.equals(Constants.TopologyType.DISPATCHER)) {
/**
* dispatcher配置
*/
conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zookeeper);
conf.put(com.creditease.dbus.commons.Constants.TOPOLOGY_ID, dispatcherTopologyId);
logger.info(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS + "=" + zookeeper);
logger.info(com.creditease.dbus.commons.Constants.TOPOLOGY_ID + "=" + dispatcherTopologyId);
}
// 启动类型为all，或者appender
if (topologyType.equals(Constants.TopologyType.ALL) || topologyType.equals(Constants.TopologyType.APPENDER)) {
/**
* appender配置
*/
conf.put(Constants.StormConfigKey.TOPOLOGY_ID, appenderTopologyId);
conf.put(Constants.StormConfigKey.ZKCONNECT, zookeeper);
conf.put(Constants.StormConfigKey.DATASOURCE, datasource);
}
// conf.put(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, 4096);
// conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, 4096);
// conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, 4096);
conf.setDebug(true);
conf.setNumAckers(1);
//设置worker数
conf.setNumWorkers(1);
//设置任务在发出后，但还没处理完成的中间状态任务的最大数量, 如果没有设置最大值为50
int MaxSpoutPending = getConfigureValueWithDefault(Constants.ConfigureKey.MAX_SPOUT_PENDING, 50);
conf.setMaxSpoutPending(MaxSpoutPending);
//设置任务在多久之内没处理完成，则这个任务处理失败
conf.setMessageTimeoutSecs(120);
String opts = getWorkerChildopts();
if (opts != null && opts.trim().length() > 0) {
conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, opts);
}
// conf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
// conf.registerSerialization(org.apache.avro.util.Utf8.class);
// conf.registerSerialization(com.creditease.dbus.commons.DBusConsumerRecord.class);
// conf.registerSerialization(org.apache.kafka.common.record.TimestampType.class);
// conf.registerSerialization(com.creditease.dbus.stream.common.appender.bean.EmitData.class);
// conf.registerSerialization(com.creditease.dbus.stream.common.appender.enums.Command.class);
// conf.registerSerialization(org.apache.avro.generic.GenericData.class);
// conf.registerSerialization(com.creditease.dbus.stream.oracle.appender.avro.GenericData.class);
// conf.registerSerialization(com.creditease.dbus.commons.DbusMessage12.class);
// conf.registerSerialization(com.creditease.dbus.commons.DbusMessage12.Schema12.class);
// conf.registerSerialization(com.creditease.dbus.commons.DbusMessage13.Schema13.class);
// conf.registerSerialization(com.creditease.dbus.commons.DbusMessage13.class);
// conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Field.class);
// conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Payload.class);
// conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Protocol.class);
// conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.ProtocolType.class);
// conf.registerSerialization(com.creditease.dbus.stream.oracle.appender.bolt.processor.appender.OraWrapperData.class);
// conf.registerSerialization(com.creditease.dbus.stream.common.appender.spout.cmds.TopicResumeCmd.class);
if (runAsLocal) {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology(topologyId, conf, topology);
/*String cmd;
do {
cmd = System.console().readLine();
} while (!cmd.equals("exit"));
cluster.shutdown();*/
} else {
StormSubmitter.submitTopology(topologyId, conf, topology);
}
}
Example 6
| Source File: AdvertisingTopology.java From streaming-benchmarks with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
Options opts = new Options();
opts.addOption("conf", true, "Path to the config file.");
CommandLineParser parser = new DefaultParser();
CommandLine cmd = parser.parse(opts, args);
String configPath = cmd.getOptionValue("conf");
Map commonConfig = Utils.findAndReadConfigFile(configPath, true);
String zkServerHosts = joinHosts((List<String>)commonConfig.get("zookeeper.servers"),
Integer.toString((Integer)commonConfig.get("zookeeper.port")));
String redisServerHost = (String)commonConfig.get("redis.host");
String kafkaTopic = (String)commonConfig.get("kafka.topic");
int kafkaPartitions = ((Number)commonConfig.get("kafka.partitions")).intValue();
int workers = ((Number)commonConfig.get("storm.workers")).intValue();
int ackers = ((Number)commonConfig.get("storm.ackers")).intValue();
int cores = ((Number)commonConfig.get("process.cores")).intValue();
int parallel = Math.max(1, cores/7);
ZkHosts hosts = new ZkHosts(zkServerHosts);
SpoutConfig spoutConfig = new SpoutConfig(hosts, kafkaTopic, "/" + kafkaTopic, UUID.randomUUID().toString());
spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
builder.setSpout("ads", kafkaSpout, kafkaPartitions);
builder.setBolt("event_deserializer", new DeserializeBolt(), parallel).shuffleGrouping("ads");
builder.setBolt("event_filter", new EventFilterBolt(), parallel).shuffleGrouping("event_deserializer");
builder.setBolt("event_projection", new EventProjectionBolt(), parallel).shuffleGrouping("event_filter");
builder.setBolt("redis_join", new RedisJoinBolt(redisServerHost), parallel).shuffleGrouping("event_projection");
builder.setBolt("campaign_processor", new CampaignProcessor(redisServerHost), parallel*2)
.fieldsGrouping("redis_join", new Fields("campaign_id"));
Config conf = new Config();
if (args != null && args.length > 0) {
conf.setNumWorkers(workers);
conf.setNumAckers(ackers);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
}
else {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("test", conf, builder.createTopology());
org.apache.storm.utils.Utils.sleep(10000);
cluster.killTopology("test");
cluster.shutdown();
}
}
