Java Code Examples for backtype.storm.LocalCluster#killTopology()
The following examples show how to use
backtype.storm.LocalCluster#killTopology() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JStormHelper.java From jstorm with Apache License 2.0 | 6 votes |
public static void runTopologyLocally(StormTopology topology, String topologyName, Config conf, int runtimeInSeconds, Callback callback) throws Exception { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyName, conf, topology); if (runtimeInSeconds < 120) { JStormUtils.sleepMs(120 * 1000); } else { JStormUtils.sleepMs(runtimeInSeconds * 1000); } if (callback != null) { callback.execute(topologyName); } cluster.killTopology(topologyName); cluster.shutdown(); }
Example 2
Source File: WordCountTrident.java From storm-hbase with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(5); if (args.length == 1) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology("wordCounter", conf, buildTopology(args[0])); Thread.sleep(60 * 1000); cluster.killTopology("wordCounter"); cluster.shutdown(); System.exit(0); } else if(args.length == 2) { conf.setNumWorkers(3); StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0])); } else{ System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]"); } }
Example 3
Source File: SequenceTopology.java From jstorm with Apache License 2.0 | 5 votes |
public static void SetLocalTopology() throws Exception { TopologyBuilder builder = new TopologyBuilder(); conf.put(TOPOLOGY_BOLT_PARALLELISM_HINT, 1); SetBuilder(builder, conf); LOG.debug("test"); LOG.info("Submit log"); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("SplitMerge", conf, builder.createTopology()); Thread.sleep(60000); cluster.killTopology("SplitMerge"); cluster.shutdown(); }
Example 4
Source File: StreamingApp.java From storm-solr with Apache License 2.0 | 5 votes |
/** * Builds and runs a StormTopology in the configured environment (development|staging|production) */ public void run() throws Exception { log.info(String.format("Running %s in %s mode.", topo.getName(), env)); String topologyName = topo.getName(); if (ENV.development == env) { int localRunSecs = Integer.parseInt(cli.getOptionValue("localRunSecs", "30")); try { LocalCluster cluster = new LocalCluster(); stormConf.put("topology.tick.tuple.freq.secs", 5); cluster.submitTopology(topologyName, stormConf, topo.build(this)); log.info("Submitted " + topologyName + " to LocalCluster at " + timestamp() + " ... sleeping for " + localRunSecs + " seconds before terminating."); try { Thread.sleep(localRunSecs * 1000); } catch (InterruptedException ie) { Thread.interrupted(); } log.info("Killing " + topologyName); cluster.killTopology(topologyName); cluster.shutdown(); log.info("Shut down LocalCluster at " + timestamp()); } catch (Exception exc) { Throwable rootCause = getRootCause(exc); log.error("Storm topology " + topologyName + " failed due to: " + rootCause, rootCause); throw exc; } finally { cleanup(); } System.exit(0); } else { StormSubmitter.submitTopology(topologyName, stormConf, topo.build(this)); } }
Example 5
Source File: StormRunner.java From flink-perf with Apache License 2.0 | 5 votes |
public static void runTopologyLocally(StormTopology topology, String topologyName, Config conf, int runtimeInSeconds) throws InterruptedException { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyName, conf, topology); Thread.sleep((long) runtimeInSeconds * MILLIS_IN_SEC); cluster.killTopology(topologyName); cluster.shutdown(); }
Example 6
Source File: LocalTopologyRunner.java From C2-Github-commit-count with MIT License | 5 votes |
public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("commit-feed-listener", new CommitFeedListener()); builder .setBolt("email-extractor", new EmailExtractor()) .shuffleGrouping("commit-feed-listener"); builder .setBolt("email-counter", new EmailCounter()) .fieldsGrouping("email-extractor", new Fields("email")); Config config = new Config(); config.setDebug(true); StormTopology topology = builder.createTopology(); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("github-commit-count-topology", config, topology); Utils.sleep(TEN_MINUTES); cluster.killTopology("github-commit-count-topology"); cluster.shutdown(); }
Example 7
Source File: WordCountTopologyBroken.java From storm-example with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { SentenceSpout spout = new SentenceSpout(); SplitSentenceBolt splitBolt = new SplitSentenceBolt(); WordCountBolt countBolt = new WordCountBolt(); ReportBolt reportBolt = new ReportBolt(); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout, 2); // SentenceSpout --> SplitSentenceBolt builder.setBolt(SPLIT_BOLT_ID, splitBolt, 2) .setNumTasks(4) .shuffleGrouping(SENTENCE_SPOUT_ID); // SplitSentenceBolt --> WordCountBolt builder.setBolt(COUNT_BOLT_ID, countBolt, 4) .shuffleGrouping(SPLIT_BOLT_ID); // WordCountBolt --> ReportBolt builder.setBolt(REPORT_BOLT_ID, reportBolt) .globalGrouping(COUNT_BOLT_ID); Config config = new Config(); config.setNumWorkers(2); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(10); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); }
Example 8
Source File: WordCountTopology.java From storm-example with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { SentenceSpout spout = new SentenceSpout(); SplitSentenceBolt splitBolt = new SplitSentenceBolt(); WordCountBolt countBolt = new WordCountBolt(); ReportBolt reportBolt = new ReportBolt(); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout, 2); // SentenceSpout --> SplitSentenceBolt builder.setBolt(SPLIT_BOLT_ID, splitBolt, 2) .setNumTasks(4) .shuffleGrouping(SENTENCE_SPOUT_ID); // SplitSentenceBolt --> WordCountBolt builder.setBolt(COUNT_BOLT_ID, countBolt, 4) .fieldsGrouping(SPLIT_BOLT_ID, new Fields("word")); // WordCountBolt --> ReportBolt builder.setBolt(REPORT_BOLT_ID, reportBolt) .globalGrouping(COUNT_BOLT_ID); Config config = new Config(); config.setNumWorkers(2); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(10); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); }
Example 9
Source File: WordCountTopology.java From storm-example with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { SentenceSpout spout = new SentenceSpout(); SplitSentenceBolt splitBolt = new SplitSentenceBolt(); WordCountBolt countBolt = new WordCountBolt(); ReportBolt reportBolt = new ReportBolt(); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout, 2); // SentenceSpout --> SplitSentenceBolt builder.setBolt(SPLIT_BOLT_ID, splitBolt, 2) .setNumTasks(4) .shuffleGrouping(SENTENCE_SPOUT_ID); // SplitSentenceBolt --> WordCountBolt builder.setBolt(COUNT_BOLT_ID, countBolt, 4) .fieldsGrouping(SPLIT_BOLT_ID, new Fields("word")); // WordCountBolt --> ReportBolt builder.setBolt(REPORT_BOLT_ID, reportBolt) .globalGrouping(COUNT_BOLT_ID); Config config = new Config(); config.setNumWorkers(2); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(10); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); }
Example 10
Source File: WordCountTopology.java From storm-example with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { SentenceSpout spout = new SentenceSpout(); SplitSentenceBolt splitBolt = new SplitSentenceBolt(); WordCountBolt countBolt = new WordCountBolt(); ReportBolt reportBolt = new ReportBolt(); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout, 2); // SentenceSpout --> SplitSentenceBolt builder.setBolt(SPLIT_BOLT_ID, splitBolt, 2) .setNumTasks(4) .shuffleGrouping(SENTENCE_SPOUT_ID); // SplitSentenceBolt --> WordCountBolt builder.setBolt(COUNT_BOLT_ID, countBolt, 4) .fieldsGrouping(SPLIT_BOLT_ID, new Fields("word")); // WordCountBolt --> ReportBolt builder.setBolt(REPORT_BOLT_ID, reportBolt) .globalGrouping(COUNT_BOLT_ID); Config config = new Config(); config.setNumWorkers(2); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(10); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); }
Example 11
Source File: WordCountTopology.java From storm-example with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { SentenceSpout spout = new SentenceSpout(); SplitSentenceBolt splitBolt = new SplitSentenceBolt(); WordCountBolt countBolt = new WordCountBolt(); ReportBolt reportBolt = new ReportBolt(); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout); // SentenceSpout --> SplitSentenceBolt builder.setBolt(SPLIT_BOLT_ID, splitBolt) .shuffleGrouping(SENTENCE_SPOUT_ID); // SplitSentenceBolt --> WordCountBolt builder.setBolt(COUNT_BOLT_ID, countBolt) .fieldsGrouping(SPLIT_BOLT_ID, new Fields("word")); // WordCountBolt --> ReportBolt builder.setBolt(REPORT_BOLT_ID, reportBolt) .globalGrouping(COUNT_BOLT_ID); Config config = new Config(); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(10); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); }
Example 12
Source File: LocalTopologyRunner.java From yuzhouwan with Apache License 2.0 | 5 votes |
public static void main(String[] args) { StormTopology topology = CreditCardTopologyBuilder.build(); Config config = new Config(); config.setDebug(true); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("local-topology", config, topology); Utils.sleep(30000); cluster.killTopology("local-topology"); cluster.shutdown(); }
Example 13
Source File: PersistentWordCount.java From storm-hbase with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws Exception { Config config = new Config(); Map<String, Object> hbConf = new HashMap<String, Object>(); if(args.length > 0){ hbConf.put("hbase.rootdir", args[0]); } config.put("hbase.conf", hbConf); WordSpout spout = new WordSpout(); WordCounter bolt = new WordCounter(); SimpleHBaseMapper mapper = new SimpleHBaseMapper() .withRowKeyField("word") .withColumnFields(new Fields("word")) .withCounterFields(new Fields("count")) .withColumnFamily("cf"); HBaseBolt hbase = new HBaseBolt("WordCount", mapper) .withConfigKey("hbase.conf"); // wordSpout ==> countBolt ==> HBaseBolt TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(COUNT_BOLT, bolt, 1).shuffleGrouping(WORD_SPOUT); builder.setBolt(HBASE_BOLT, hbase, 1).fieldsGrouping(COUNT_BOLT, new Fields("word")); if (args.length == 1) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology("test", config, builder.createTopology()); Thread.sleep(30000); cluster.killTopology("test"); cluster.shutdown(); System.exit(0); } else if (args.length == 2) { StormSubmitter.submitTopology(args[1], config, builder.createTopology()); } else{ System.out.println("Usage: HdfsFileTopology <hdfs url> [topology name]"); } }
Example 14
Source File: LookupWordCount.java From storm-hbase with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws Exception { Config config = new Config(); Map<String, Object> hbConf = new HashMap<String, Object>(); if(args.length > 0){ hbConf.put("hbase.rootdir", args[0]); } config.put("hbase.conf", hbConf); WordSpout spout = new WordSpout(); TotalWordCounter totalBolt = new TotalWordCounter(); SimpleHBaseMapper mapper = new SimpleHBaseMapper().withRowKeyField("word"); HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria(); projectionCriteria.addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count")); WordCountValueMapper rowToTupleMapper = new WordCountValueMapper(); HBaseLookupBolt hBaseLookupBolt = new HBaseLookupBolt("WordCount", mapper, rowToTupleMapper) .withConfigKey("hbase.conf") .withProjectionCriteria(projectionCriteria); //wordspout -> lookupbolt -> totalCountBolt TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(LOOKUP_BOLT, hBaseLookupBolt, 1).shuffleGrouping(WORD_SPOUT); builder.setBolt(TOTAL_COUNT_BOLT, totalBolt, 1).fieldsGrouping(LOOKUP_BOLT, new Fields("columnName")); if (args.length == 1) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology("test", config, builder.createTopology()); Thread.sleep(30000); cluster.killTopology("test"); cluster.shutdown(); System.exit(0); } else if (args.length == 2) { StormSubmitter.submitTopology(args[1], config, builder.createTopology()); } else{ System.out.println("Usage: LookupWordCount <hbase.rootdir>"); } }
Example 15
Source File: SequenceFileTopology.java From storm-hdfs with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws Exception { Config config = new Config(); config.setNumWorkers(1); SentenceSpout spout = new SentenceSpout(); // sync the filesystem after every 1k tuples SyncPolicy syncPolicy = new CountSyncPolicy(1000); // rotate files when they reach 5MB FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("/source/") .withExtension(".seq"); // create sequence format instance. DefaultSequenceFormat format = new DefaultSequenceFormat("timestamp", "sentence"); SequenceFileBolt bolt = new SequenceFileBolt() .withFsUrl(args[0]) .withFileNameFormat(fileNameFormat) .withSequenceFormat(format) .withRotationPolicy(rotationPolicy) .withSyncPolicy(syncPolicy) .withCompressionType(SequenceFile.CompressionType.RECORD) .withCompressionCodec("deflate") .addRotationAction(new MoveFileAction().toDestination("/dest/")); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout, 1); // SentenceSpout --> MyBolt builder.setBolt(BOLT_ID, bolt, 4) .shuffleGrouping(SENTENCE_SPOUT_ID); if (args.length == 1) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(120); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); System.exit(0); } else if(args.length == 2) { StormSubmitter.submitTopology(args[1], config, builder.createTopology()); } }
Example 16
Source File: HdfsFileTopology.java From storm-hdfs with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws Exception { Config config = new Config(); config.setNumWorkers(1); SentenceSpout spout = new SentenceSpout(); // sync the filesystem after every 1k tuples SyncPolicy syncPolicy = new CountSyncPolicy(1000); // rotate files when they reach 5MB FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("/foo/") .withExtension(".txt"); // use "|" instead of "," for field delimiter RecordFormat format = new DelimitedRecordFormat() .withFieldDelimiter("|"); Yaml yaml = new Yaml(); InputStream in = new FileInputStream(args[1]); Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in); in.close(); config.put("hdfs.config", yamlConf); HdfsBolt bolt = new HdfsBolt() .withConfigKey("hdfs.config") .withFsUrl(args[0]) .withFileNameFormat(fileNameFormat) .withRecordFormat(format) .withRotationPolicy(rotationPolicy) .withSyncPolicy(syncPolicy) .addRotationAction(new MoveFileAction().toDestination("/dest2/")); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout, 1); // SentenceSpout --> MyBolt builder.setBolt(BOLT_ID, bolt, 4) .shuffleGrouping(SENTENCE_SPOUT_ID); if (args.length == 2) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(120); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); System.exit(0); } else if (args.length == 3) { StormSubmitter.submitTopology(args[0], config, builder.createTopology()); } else{ System.out.println("Usage: HdfsFileTopology [topology name] <yaml config file>"); } }
Example 17
Source File: SentimentAnalysisTopology.java From StormTweetsSentimentD3Viz with Apache License 2.0 | 4 votes |
public static final void main(final String[] args) throws Exception { final ApplicationContext applicationContext = new ClassPathXmlApplicationContext("applicationContext.xml"); final JmsProvider jmsProvider = new SpringJmsProvider(applicationContext, "jmsConnectionFactory", "notificationQueue"); final TopologyBuilder topologyBuilder = new TopologyBuilder(); final JmsBolt jmsBolt = new JmsBolt(); jmsBolt.setJmsProvider(jmsProvider); jmsBolt.setJmsMessageProducer((session, input) -> { final String json = "{\"stateCode\":\"" + input.getString(0) + "\", \"sentiment\":" + input.getInteger(1) + "}"; return session.createTextMessage(json); }); try { final Config config = new Config(); config.setMessageTimeoutSecs(120); config.setDebug(true); topologyBuilder.setSpout("twitterspout", new TwitterSpout()); topologyBuilder.setBolt("statelocatorbolt", new StateLocatorBolt()) .shuffleGrouping("twitterspout"); topologyBuilder.setBolt("sentimentcalculatorbolt", new SentimentCalculatorBolt()) .fieldsGrouping("statelocatorbolt", new Fields("state")); topologyBuilder.setBolt("jmsBolt", jmsBolt).fieldsGrouping("sentimentcalculatorbolt", new Fields("stateCode")); //Submit it to the cluster, or submit it locally if (null != args && 0 < args.length) { config.setNumWorkers(3); StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology()); } else { config.setMaxTaskParallelism(10); final LocalCluster localCluster = new LocalCluster(); localCluster.submitTopology(Constants.TOPOLOGY_NAME, config, topologyBuilder.createTopology()); //Run this topology for 600 seconds so that we can complete processing of decent # of tweets. Utils.sleep(600 * 1000); LOGGER.info("Shutting down the cluster..."); localCluster.killTopology(Constants.TOPOLOGY_NAME); localCluster.shutdown(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { LOGGER.info("Shutting down the cluster..."); localCluster.killTopology(Constants.TOPOLOGY_NAME); localCluster.shutdown(); } }); } } catch (final Exception exception) { //Deliberate no op; exception.printStackTrace(); } LOGGER.info("\n\n\n\t\t*****Please clean your temp folder \"{}\" now!!!*****", System.getProperty("java.io.tmpdir")); }
Example 18
Source File: AuditActiveLoginsTopology.java From Kafka-Storm-ElasticSearch with Apache License 2.0 | 4 votes |
private static void loadTopologyPropertiesAndSubmit(Properties properties, Config config) throws Exception { String stormExecutionMode = properties.getProperty("storm.execution.mode","local"); int stormWorkersNumber = Integer.parseInt(properties.getProperty("storm.workers.number","2")); int maxTaskParallism = Integer.parseInt(properties.getProperty("storm.max.task.parallelism","2")); String topologyName = properties.getProperty("storm.topology.name","topologyName"); String zookeeperHosts = properties.getProperty("zookeeper.hosts"); int topologyBatchEmitMillis = Integer.parseInt( properties.getProperty("storm.topology.batch.interval.miliseconds","2000")); // How often a batch can be emitted in a Trident topology. config.put(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS, topologyBatchEmitMillis); config.setNumWorkers(stormWorkersNumber); config.setMaxTaskParallelism(maxTaskParallism); AuditActiveLoginsTopology auditActiveLoginsTopology = new AuditActiveLoginsTopology(zookeeperHosts); StormTopology stormTopology = auditActiveLoginsTopology.buildTopology(properties); // Elastic Search specific properties config.put(StormElasticSearchConstants.ES_HOST, properties.getProperty("elasticsearch.host", "localhost")); config.put(StormElasticSearchConstants.ES_PORT, (Integer.parseInt(properties.getProperty("elasticsearch.port", "9300")))); config.put(StormElasticSearchConstants.ES_CLUSTER_NAME, properties.getProperty("elasticsearch.cluster.name")); config.put("elasticsearch.index", properties.getProperty("elasticsearch.index")); config.put("elasticsearch.type", properties.getProperty("elasticsearch.type")); switch (stormExecutionMode){ case ("cluster"): String nimbusHost = properties.getProperty("storm.nimbus.host","localhost"); String nimbusPort = properties.getProperty("storm.nimbus.port","6627"); config.put(Config.NIMBUS_HOST, nimbusHost); config.put(Config.NIMBUS_THRIFT_PORT, Integer.parseInt(nimbusPort)); config.put(Config.STORM_ZOOKEEPER_PORT, parseZkPort(zookeeperHosts)); config.put(Config.STORM_ZOOKEEPER_SERVERS, parseZkHosts(zookeeperHosts)); StormSubmitter.submitTopology(topologyName, config, stormTopology); break; case ("local"): default: int localTimeExecution = Integer.parseInt(properties.getProperty("storm.local.execution.time","20000")); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyName, config, stormTopology); Thread.sleep(localTimeExecution); cluster.killTopology(topologyName); cluster.shutdown(); System.exit(0); } }