Java Code Examples for backtype.storm.LocalCluster#submitTopology()

private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(String.valueOf(conf.get("topology.name")), conf, builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		e.printStackTrace();
	}
}
 

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(60 * 1000);
        cluster.killTopology("wordCounter");
        cluster.shutdown();
        System.exit(0);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}
 

public static void main(String[] args) throws Exception {
  Config conf = new Config();
  conf.setMaxSpoutPending(20);
  if (args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
    for (int i = 0; i < 100; i++) {
      System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
      Thread.sleep(1000);
    }
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
  }
}
 

public static void main(String[] args) throws Exception {
        Config conf = new Config();
//        conf.put(Config.TOPOLOGY_DEBUG,true);
        LocalCluster cluster = new LocalCluster();

        // This time we use a "FeederBatchSpout", a spout designed for testing.
        FeederBatchSpout testSpout = new FeederBatchSpout(ImmutableList.of("name", "city", "age"));
        cluster.submitTopology("advanced_primitives", conf, advancedPrimitives(testSpout));

        // You can "hand feed" values to the topology by using this spout
        testSpout.feed(ImmutableList.of(new Values("rose", "Shanghai", 32), new Values("mary", "Shanghai", 51), new Values("pere", "Jakarta", 65), new Values("Tom", "Jakarta", 10)));
    }
 

public static void main(String[] args) throws Exception {
    if (getProperties() == null || getProperties().isEmpty()) {
        System.out.println("Property file <ignite-storm.property> is not found or empty");
        return;
    }
    // Ignite Stream Ibolt
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();

    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName(getProperties().getProperty("cache.name"));

    stormStreamer.setIgniteTupleField(getProperties().getProperty("tuple.name"));
    stormStreamer.setIgniteConfigFile(getProperties().getProperty("ignite.spring.xml"));


    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new FileSourceSpout(), 1);
    builder.setBolt("limit", new SpeedLimitBolt(), 1).fieldsGrouping("spout", new Fields("trafficLog"));
    // set ignite bolt
    builder.setBolt("ignite-bolt", stormStreamer, STORM_EXECUTORS).shuffleGrouping("limit");

    Config conf = new Config();
    conf.setDebug(false);

    conf.setMaxTaskParallelism(1);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("speed-violation", conf, builder.createTopology());
    Thread.sleep(10000);
    cluster.shutdown();

}
 

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(120 * 1000);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}
 

@Test
public void test_transaction_word() {
    try {
        MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
        TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
        builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
        builder.setBolt("bucketize", new Bucketize()).shuffleGrouping("count");
        builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));

        LocalCluster cluster = new LocalCluster();

        Config config = new Config();
        config.setDebug(true);
        config.setMaxSpoutPending(3);

        cluster.submitTopology("top-n-topology", config, builder.buildTopology());

        JStormUtils.sleepMs(60 * 1000);
        
        
        assertEquals(false, outOfOrder.get() );
        assertNotSame(0, receiveCounter1.get());
        assertNotSame(0, receiveCounter2.get());
        
        //cluster.killTopology("top-n-topology");
        //cluster.shutdown();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Failed to run simple transaction");
    }

}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	int par = pt.getInt("para");


	TridentTopology topology = new TridentTopology();
	Stream sourceStream = topology.newStream("source", new Generator(pt)).parallelismHint(pt.getInt("sourceParallelism"));

	Stream repart = sourceStream.partitionBy(new Fields("id"));
	for(int i = 0; i < pt.getInt("repartitions", 1) - 1; i++) {
		repart = repart.each(new Fields("id"), new IdentityEach(), new Fields("id"+i)).partitionBy(new Fields("id"+i));
	}
	repart.each(new Fields("id", "host", "time", "payload"), new Sink(pt), new Fields("dontcare")).parallelismHint(pt.getInt("sinkParallelism"));

	Config conf = new Config();
	conf.setDebug(false);

	if (!pt.has("local")) {
		conf.setNumWorkers(par);

		StormSubmitter.submitTopologyWithProgressBar("throughput-"+pt.get("name", "no_name"), conf, topology.build());
	}
	else {
		conf.setMaxTaskParallelism(par);

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("throughput", conf, topology.build());

		Thread.sleep(30000);

		cluster.shutdown();
	}

}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	int par = pt.getInt("para");

	TopologyBuilder builder = new TopologyBuilder();

	builder.setSpout("source0", new Generator(pt), pt.getInt("sourceParallelism"));
	int i = 0;
	for(; i < pt.getInt("repartitions", 1) - 1;i++) {
		System.out.println("adding source"+i+" --> source"+(i+1));
		builder.setBolt("source"+(i+1), new RepartPassThroughBolt(pt), pt.getInt("sinkParallelism")).fieldsGrouping("source" + i, new Fields("id"));
	}
	System.out.println("adding final source"+i+" --> sink");

	builder.setBolt("sink", new Sink(pt), pt.getInt("sinkParallelism")).fieldsGrouping("source"+i, new Fields("id"));


	Config conf = new Config();
	conf.setDebug(false);
	//System.exit(1);

	if (!pt.has("local")) {
		conf.setNumWorkers(par);

		StormSubmitter.submitTopologyWithProgressBar("throughput-"+pt.get("name", "no_name"), conf, builder.createTopology());
	}
	else {
		conf.setMaxTaskParallelism(par);

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("throughput", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}

}
 

public static void main(String[] args) {

        StormTopology topology = CreditCardTopologyBuilder.build();
        Config config = new Config();
        config.setDebug(true);

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("local-topology", config, topology);
        Utils.sleep(30000);

        cluster.killTopology("local-topology");
        cluster.shutdown();
    }
 

@Test
public void testFilter_nothingPasses() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key3", "val50").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}
 

public static void main(String[] args) throws Exception {

        SentenceSpout spout = new SentenceSpout();
        SplitSentenceBolt splitBolt = new SplitSentenceBolt();
        WordCountBolt countBolt = new WordCountBolt();
        ReportBolt reportBolt = new ReportBolt();


        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout(SENTENCE_SPOUT_ID, spout, 2);
        // SentenceSpout --> SplitSentenceBolt
        builder.setBolt(SPLIT_BOLT_ID, splitBolt, 2)
                .setNumTasks(4)
                .shuffleGrouping(SENTENCE_SPOUT_ID);
        // SplitSentenceBolt --> WordCountBolt
        builder.setBolt(COUNT_BOLT_ID, countBolt, 4)
                .fieldsGrouping(SPLIT_BOLT_ID, new Fields("word"));
        // WordCountBolt --> ReportBolt
        builder.setBolt(REPORT_BOLT_ID, reportBolt)
                .globalGrouping(COUNT_BOLT_ID);

        Config config = new Config();
        config.setNumWorkers(2);

        LocalCluster cluster = new LocalCluster();

        cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
        waitForSeconds(10);
        cluster.killTopology(TOPOLOGY_NAME);
        cluster.shutdown();
    }
 

public static void main(String[] args) 
    throws Exception {
  
  Config conf = new Config();
  LocalCluster cluster = new LocalCluster();
  
  TridentTopology topology = new TridentTopology();

  Stream movingAvgStream =
    topology.newStream("ticks-spout", buildSpout())
    .each(new Fields("stock-ticks"), new TickParser(), new Fields("price"))
    .aggregate(new Fields("price"), new CalculateAverage(), new Fields("count"));

  cluster.submitTopology("moving-avg", conf, topology.build());
}
 

public static void main(String[] args) throws Exception {
        Config conf = new Config();
//        conf.put(Config.TOPOLOGY_DEBUG,true);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("advanced_primitives", conf, advancedPrimitives(new FakeTweetsBatchSpout(1000)));
        Thread.sleep(30000);
        cluster.shutdown();
    }
 

@Test
public void test_timeTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.TIME, 2)
          .evict(Policy.COUNT, 10)
          .clearOnTrigger()
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);


  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(4, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 10);
  }
}
 

private static void loadTopologyPropertiesAndSubmit(Properties properties,
		Config config) throws Exception {
	
	String stormExecutionMode = properties.getProperty("storm.execution.mode","local");
	int stormWorkersNumber = Integer.parseInt(properties.getProperty("storm.workers.number","2"));
	int maxTaskParallism = Integer.parseInt(properties.getProperty("storm.max.task.parallelism","2"));
	String topologyName = properties.getProperty("storm.topology.name","topologyName");
	String zookeeperHosts = properties.getProperty("zookeeper.hosts");
	int topologyBatchEmitMillis = Integer.parseInt(
			properties.getProperty("storm.topology.batch.interval.miliseconds","2000"));
	
	// How often a batch can be emitted in a Trident topology.
	config.put(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS, topologyBatchEmitMillis);
	config.setNumWorkers(stormWorkersNumber);
	config.setMaxTaskParallelism(maxTaskParallism);
	
	AuditActiveLoginsTopology auditActiveLoginsTopology = new AuditActiveLoginsTopology(zookeeperHosts);
	StormTopology stormTopology = auditActiveLoginsTopology.buildTopology(properties);
	
	// Elastic Search specific properties
	config.put(StormElasticSearchConstants.ES_HOST, properties.getProperty("elasticsearch.host", "localhost"));
	config.put(StormElasticSearchConstants.ES_PORT, (Integer.parseInt(properties.getProperty("elasticsearch.port", "9300"))));
	config.put(StormElasticSearchConstants.ES_CLUSTER_NAME, properties.getProperty("elasticsearch.cluster.name"));
	config.put("elasticsearch.index", properties.getProperty("elasticsearch.index"));
	config.put("elasticsearch.type", properties.getProperty("elasticsearch.type"));

	switch (stormExecutionMode){
		case ("cluster"):
			String nimbusHost = properties.getProperty("storm.nimbus.host","localhost");
			String nimbusPort = properties.getProperty("storm.nimbus.port","6627");
			config.put(Config.NIMBUS_HOST, nimbusHost);
			config.put(Config.NIMBUS_THRIFT_PORT, Integer.parseInt(nimbusPort));
			config.put(Config.STORM_ZOOKEEPER_PORT, parseZkPort(zookeeperHosts));
			config.put(Config.STORM_ZOOKEEPER_SERVERS, parseZkHosts(zookeeperHosts));
			StormSubmitter.submitTopology(topologyName, config, stormTopology);
			break;
		case ("local"):
		default:
			int localTimeExecution = Integer.parseInt(properties.getProperty("storm.local.execution.time","20000"));
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology(topologyName, config, stormTopology);
			Thread.sleep(localTimeExecution);
			cluster.killTopology(topologyName);
			cluster.shutdown();
			System.exit(0);
	}		
}
 

/**
 * @param args
 */
public static void main(String[] args) 
{
	// first some global (topology configuration)
	StormCVConfig conf = new StormCVConfig();

	/**
	 * Sets the OpenCV library to be used which depends on the system the topology is being executed on
	 */
	//conf.put( StormCVConfig.STORMCV_OPENCV_LIB, "mac64_opencv_java248.dylib" );

	conf.setNumWorkers( 4 );                                           // number of workers in the topology
	conf.setMaxSpoutPending( 32 );                                     // maximum un-acked/un-failed frames per spout (spout blocks if this number is reached)
	conf.put( StormCVConfig.STORMCV_FRAME_ENCODING, Frame.JPG_IMAGE ); // indicates frames will be encoded as JPG throughout the topology (JPG is the default when not explicitly set)
	conf.put( Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true );         // True if Storm should timeout messages or not.
	conf.put( Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS , 10 );             // The maximum amount of time given to the topology to fully process a message emitted by a spout (default = 30)
	conf.put( StormCVConfig.STORMCV_SPOUT_FAULTTOLERANT, false );      // indicates if the spout must be fault tolerant; i.e. spouts do NOT! replay tuples on fail
	conf.put( StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC, 30 );          // TTL (seconds) for all elements in all caches throughout the topology (avoids memory overload)

	// some live camera feeds from http://webcam.prvgld.nl/
	List<String> urls = new ArrayList<String>();
	urls.add( "rtsp://streaming3.webcam.nl:1935/n224/n224.stream" );
	urls.add( "rtsp://streaming3.webcam.nl:1935/n233/n233.stream" );

	int frameSkip = 13;

	// now create the topology itself (spout -> contrast enhancement --> streamer)
	TopologyBuilder builder = new TopologyBuilder();

	// just one spout reading streams; i.e. this spout reads two streams in parallel
	builder.setSpout( "spout", new CVParticleSpout( new StreamFrameFetcher( urls ).frameSkip( frameSkip ) ), 1 );

	// add bolt that does contrast enhancement (choose HSV_EQUALIZE_HIST or GRAY_EQUALIZE_HIST as algorithm)
	builder.setBolt( "contrastenhancement", new SingleInputBolt( new GlobalContrastEnhancementOp().setAlgorithm( CEAlgorithm.HSV_EQUALIZE_HIST ) ), 1 )
	.shuffleGrouping( "spout" );

	// add bolt that creates a web service on port 8558 enabling users to view the result
	builder.setBolt( "streamer", new BatchInputBolt(
			new SlidingWindowBatcher( 2, frameSkip).maxSize( 6 ), // note the required batcher used as a buffer and maintains the order of the frames
			new MjpegStreamingOp().port( 8558 ).framerate( 5 ) ).groupBy( new Fields( FrameSerializer.STREAMID ) )
			, 1)
			.shuffleGrouping( "contrastenhancement" );

	// NOTE: if the topology is started (locally) go to http://localhost:8558/streaming/tiles and click the image to see the stream!

	try 
	{	
		// run in local mode
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology( "BackgroundSubtraction", conf, builder.createTopology() );
		Utils.sleep( 120 * 1000 ); // run for one minute and then kill the topology
		cluster.shutdown();
		System.exit( 1 );

		// run on a storm cluster
		// StormSubmitter.submitTopology("some_topology_name", conf, builder.createTopology());
	} 
	catch (Exception e)
	{
		e.printStackTrace();
	}
}
 

@Test
public void test_countTrigger_timeEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 600)       // trigger every 500 events received
          .evict(Policy.TIME, 1)            // only keep last 1 second in the window
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }


  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(1, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() > 90);
    assertTrue(event.<Long>get("aCount").getValue() < 100);
  }
}
 

@Test
public void test_single_join() {
	receiveCounter.set(0);
	try {
		FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
		FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));

		TopologyBuilder builder = new TopologyBuilder();
		builder.setSpout("gender", genderSpout);
		builder.setSpout("age", ageSpout);
		builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age")))
				.fieldsGrouping("gender", new Fields("id")).fieldsGrouping("age", new Fields("id"));

		Config conf = new Config();
		conf.setDebug(true);

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("join-example", conf, builder.createTopology());

		for (int i = 0; i < 10; i++) {
			String gender;
			if (i % 2 == 0) {
				gender = "male";
			} else {
				gender = "female";
			}
			genderSpout.feed(new Values(i, gender));
		}

		for (int i = 9; i >= 0; i--) {
			ageSpout.feed(new Values(i, i + 20));
		}

		JStormUtils.sleepMs(60 * 1000);

		assertNotSame(0, receiveCounter.get());

		cluster.shutdown();
	} catch (Exception e) {
		Assert.fail("Failed to run SingleJoinExample");
	}
}
 

/**
 * When run this test, please check the following through jstack and log
 * 1) for blue-spout, num of executors is 2, # of tasks is 2
 * <p>
 * Expected:
 * <p>
 * a. 2 threads uniquely named Thread-*-blue-spout-executor[*,*]
 * b. each thread will have single task
 * <p>
 * 2) for green-bolt, num of executors is 2, # of tasks is 4
 * <p>
 * Expected:
 * <p>
 * a. 2 threads uniquely named Thread-*-green-bolt-executor[*,*]
 * b. each thread will have 2 tasks
 * <p>
 * 3) for yellow-bolt, num of executors is 6, # of tasks is 6
 * <p>
 * Expected:
 * <p>
 * a. 6 threads uniquely named Thread-*-yellow-bolt-executor[*,*]
 * b. each thread will have 1 tasks
 * <p>
 * <p>
 * Continue to think:
 * <p>
 * For alter engine, if we use multiple tasks per component instead of one task per component,
 * what will the parallelism mechanism affect?
 *
 * @throws Exception
 */
@Ignore
@Test
public void testParallelism() throws Exception {
    Config conf = new Config();
    conf.setNumWorkers(2); // use two worker processes
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("blue-spout", new BlueSpout(), 2); // parallelism hint

    topologyBuilder.setBolt("green-bolt", new GreenBolt(), 2)
        .setNumTasks(4)
        .shuffleGrouping("blue-spout");

    topologyBuilder.setBolt("yellow-bolt", new YellowBolt(), 6)
        .shuffleGrouping("green-bolt");

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("mytopology", new HashMap(), topologyBuilder.createTopology());

    while (true) {
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}