backtype.storm.generated.AlreadyAliveException Java Examples

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  if (args.length < 1) {
    throw new RuntimeException("Specify topology name");
  }

  int parallelism = 1;
  if (args.length > 1) {
    parallelism = Integer.parseInt(args[1]);
  }
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("word", new WordSpout(), parallelism);
  builder.setBolt("consumer", new ConsumerBolt(), parallelism)
      .fieldsGrouping("word", new Fields("word"));
  Config conf = new Config();
  conf.setNumWorkers(parallelism);

  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 

@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public void submitTopology(String topoName,
                           Map config,
                           StormTopology stormTopology)
    throws AlreadyAliveException, InvalidTopologyException {
  assertNotAlive();

  this.topologyName = topoName;
  this.conf = config;
  this.topology = stormTopology;

  simulator.submitTopology(topoName,
      ConfigUtils.translateConfig(config),
      stormTopology.getStormTopology());
}
 

private static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  LOG.info("running benchmark " + name);
  IBenchmark benchmark =  (IBenchmark) Runner.getApplicationFromName(PACKAGE + "." + name);
  Config config = new Config();
  config.putAll(Utils.readStormConfig());
  config.setDebug(true);
  StormTopology topology = benchmark.getTopology(config);
  LocalCluster localCluster = new LocalCluster();
  localCluster.submitTopology(name, config, topology);
  final int runtime = BenchmarkUtils.getInt(config, MetricsCollectorConfig.METRICS_TOTAL_TIME,
          MetricsCollectorConfig.DEFAULT_TOTAL_TIME);
  IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
  collector.run();
  try {
    Thread.sleep(runtime);
  } catch (InterruptedException e) {
    LOG.error("benchmark interrupted", e);
  }
  localCluster.shutdown();
}
 

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }

    int parallelism = 10;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism)
            .fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
/*
Set config here
*/

    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = JStormHelper.getConfig(args);
    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);
    boolean isValueSpout = JStormUtils.parseBoolean(conf.get("is.value.spout"), false);

    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isValueSpout)
        builder.setSpoutWithAck("spout", new BatchAckerValueSpout(), spoutParallelism);
    else
        builder.setSpoutWithAck("spout", new BatchAckerSpout(), spoutParallelism);
    builder.setBoltWithAck("split", new BatchAckerSplit(), splitParallelism).localOrShuffleGrouping("spout");;
    builder.setBoltWithAck("count", new BatchAckerCount(), countParallelism).fieldsGrouping("split", new Fields("word"));
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}
 

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

public void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    Config conf = getConf();
    StormTopology topology = buildTopology();
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    
    if (streamName.contains("zeromq")) {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.zeroMq.MQContext");
        
    } else {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.netty.NettyContext");
    }
    
    StormSubmitter.submitTopology(streamName, conf, topology);
    
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Map config = new Config();
    config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler");
    config.put(Config.TOPOLOGY_WORKERS, 2);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2);
    builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4);
    StormTopology topology = builder.createTopology();

    StormSubmitter.submitTopology("TMUdfTopology", config, topology);
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, UnknownHostException, InterruptedException {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	TopologyBuilder builder = new TopologyBuilder();
	BrokerHosts hosts = new ZkHosts(pt.getRequired("zookeeper"));
	SpoutConfig spoutConfig = new SpoutConfig(hosts, pt.getRequired("topic"), "/" + pt.getRequired("topic"), UUID.randomUUID().toString());
	spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
	KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
	builder.setSpout("source", kafkaSpout, pt.getInt("sourceParallelism"));

	builder.setBolt("sink", new Throughput.Sink(pt), pt.getInt("sinkParallelism")).noneGrouping("source");

	Config conf = new Config();
	conf.setDebug(false);

	if (!pt.has("local")) {
		conf.setNumWorkers(pt.getInt("par", 2));

		StormSubmitter.submitTopologyWithProgressBar("kafka-spout-"+pt.get("name", "no_name"), conf, builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(pt.getInt("par", 2));

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("kafka-spout", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}
}
 

private static void runApplication(IApplication app)
        throws AlreadyAliveException, InvalidTopologyException {
  config.putAll(Utils.readStormConfig());
  String name = (String) config.get(Config.TOPOLOGY_NAME);
  topology = app.getTopology(config);
  StormSubmitter.submitTopology(name, config, topology);
}
 

public static void runBenchmark(IBenchmark benchmark)
        throws AlreadyAliveException, InvalidTopologyException,
        ClassNotFoundException, IllegalAccessException, InstantiationException {
  runApplication(benchmark);
  if (isMetricsEnabled()) {
    IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
    collector.run();
  }
}
 

public static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  if (name.startsWith("storm.benchmark.benchmarks")) {
    LOG.info("running benchmark " + name);
    runBenchmark((IBenchmark) getApplicationFromName(name));
  } else if (name.startsWith("storm.benchmark.tools.producer")) {
    LOG.info("running producer " + name);
    runProducer((IProducer) getApplicationFromName(name));
  } else {
    throw new RuntimeException(name + " is neither benchmark nor producer");
  }
}
 

public void SetDPRCTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    // LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
    // "exclamation");
    //
    // builder.addBolt(new TotalCount(), 3);
    //
    // Config conf = new Config();
    //
    // conf.setNumWorkers(3);
    // StormSubmitter.submitTopology("rpc", conf,
    // builder.createRemoteTopology());
    System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}
 

private StormTopology createTopology()
        throws AlreadyAliveException, InvalidTopologyException {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    
    return builder.createTopology();
}
 

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    SetBuilder(builder, conf);
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
}
 

public static void SetDPRCTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    // LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
    // "exclamation");
    //
    // builder.addBolt(new TotalCount(), 3);
    //
    // Config conf = new Config();
    //
    // conf.setNumWorkers(3);
    // StormSubmitter.submitTopology("rpc", conf,
    // builder.createRemoteTopology());
    System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}
 

/**
 * check whether the topology is bActive?
 *
 * @throws Exception
 */
public void checkTopologyActive(NimbusData nimbus, String topologyName, boolean bActive) throws Exception {
    if (isTopologyActive(nimbus.getStormClusterState(), topologyName) != bActive) {
        if (bActive) {
            throw new NotAliveException(topologyName + " is not alive");
        } else {
            throw new AlreadyAliveException(topologyName + " is already alive");
        }
    }
}
 

void submitTopology(String topologyName, Map conf, StormTopology topology)
throws AlreadyAliveException, InvalidTopologyException;
 

void submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, SubmitOptions submitOpts)
throws AlreadyAliveException, InvalidTopologyException;
 

public static void main(String[] args) throws IOException{
	Properties props = StormSamoaUtils.getProperties();
	
	String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
	if(uploadedJarLocation == null){
		logger.error("Invalid properties file. It must have key {}", 
				StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
		return;
	}
	
	List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
	int numWorkers = StormSamoaUtils.numWorkers(tmpArgs);
	
	args = tmpArgs.toArray(new String[0]);
	StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);

	Config conf = new Config();
	conf.putAll(Utils.readStormConfig());
	conf.putAll(Utils.readCommandLineOpts());
	conf.setDebug(false);
	conf.setNumWorkers(numWorkers);
	
	String profilerOption = 
			props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY);
	if(profilerOption != null){
		String topoWorkerChildOpts =  (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS);
		StringBuilder optionBuilder = new StringBuilder();
		if(topoWorkerChildOpts != null){
			optionBuilder.append(topoWorkerChildOpts);	
			optionBuilder.append(' ');
		}
		optionBuilder.append(profilerOption);
		conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString());
	}

	Map<String, Object> myConfigMap = new HashMap<String, Object>(conf);
	StringWriter out = new StringWriter();

	try {
		JSONValue.writeJSONString(myConfigMap, out);
	} catch (IOException e) {
		System.out.println("Error in writing JSONString");
		e.printStackTrace();
		return;
	}
	
	Config config = new Config();
	config.putAll(Utils.readStormConfig());
	
	String nimbusHost = (String) config.get(Config.NIMBUS_HOST);
			
	NimbusClient nc = new NimbusClient(nimbusHost);
	String topologyName = stormTopo.getTopologyName();
	try {
		System.out.println("Submitting topology with name: " 
				+ topologyName);
		nc.getClient().submitTopology(topologyName, uploadedJarLocation,
				out.toString(), stormTopo.getStormBuilder().createTopology());
		System.out.println(topologyName + " is successfully submitted");

	} catch (AlreadyAliveException aae) {
		System.out.println("Fail to submit " + topologyName
				+ "\nError message: " + aae.get_msg());
	} catch (InvalidTopologyException ite) {
		System.out.println("Invalid topology for " + topologyName);
		ite.printStackTrace();
	} catch (TException te) {
		System.out.println("Texception for " + topologyName);
		te.printStackTrace();
	} 		
}
 

public static void runTopologyRemotely(StormTopology topology, String topologyName, Config conf)
    throws AlreadyAliveException, InvalidTopologyException {
  StormSubmitter.submitTopology(topologyName, conf, topology);
}
 

public static void runProducer(IProducer producer)
        throws AlreadyAliveException, InvalidTopologyException,
        ClassNotFoundException, IllegalAccessException, InstantiationException {
  runApplication(producer);
}
 

public static void main(String[] args) throws IllegalArgumentException, KeeperException, InterruptedException, AlreadyAliveException, InvalidTopologyException, IOException {
    String propertiesFile = null;
    String mode = null;
    
    if (args.length != 2) {
        printUsageAndExit();
    } else {
        propertiesFile = args[0];
        mode = args[1];
    }
    
    configure(propertiesFile);

    final KinesisSpoutConfig config =
            new KinesisSpoutConfig(streamName, zookeeperEndpoint).withZookeeperPrefix(zookeeperPrefix)
                    .withInitialPositionInStream(initialPositionInStream)
                    .withRegion(Regions.fromName(regionName));

    final KinesisSpout spout = new KinesisSpout(config, new CustomCredentialsProviderChain(), new ClientConfiguration());
    TopologyBuilder builder = new TopologyBuilder();
    LOG.info("Using Kinesis stream: " + config.getStreamName());


    // Using number of shards as the parallelism hint for the spout.
    builder.setSpout("Kinesis", spout, 2);
    builder.setBolt("Parse", new ParseReferrerBolt(), 6).shuffleGrouping("Kinesis");
    builder.setBolt("Count", new RollingCountBolt(5, 2,elasticCacheRedisEndpoint), 6).fieldsGrouping("Parse", new Fields("referrer"));

    //builder.setBolt("Count", new CountReferrerBolt(), 12).fieldsGrouping("Parse", new Fields("referrer"));

    Config topoConf = new Config();
    topoConf.setFallBackOnJavaSerialization(true);
    topoConf.setDebug(false);

    if (mode.equals("LocalMode")) {
        LOG.info("Starting sample storm topology in LocalMode ...");
        new LocalCluster().submitTopology("test_spout", topoConf, builder.createTopology());
    } else if (mode.equals("RemoteMode")) {
        topoConf.setNumWorkers(1);
        topoConf.setMaxSpoutPending(5000);
        LOG.info("Submitting sample topology " + topologyName + " to remote cluster.");
        StormSubmitter.submitTopology(topologyName, topoConf, builder.createTopology());            
    } else {
        printUsageAndExit();
    }

}
 

public static void main(String[] args) throws IOException {
  Properties props = StormSamoaUtils.getProperties();

  String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
  if (uploadedJarLocation == null) {
    logger.error("Invalid properties file. It must have key {}",
        StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
    return;
  }

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
  int numWorkers = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.putAll(Utils.readCommandLineOpts());
  conf.setDebug(false);
  conf.setNumWorkers(numWorkers);

  String profilerOption =
      props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY);
  if (profilerOption != null) {
    String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS);
    StringBuilder optionBuilder = new StringBuilder();
    if (topoWorkerChildOpts != null) {
      optionBuilder.append(topoWorkerChildOpts);
      optionBuilder.append(' ');
    }
    optionBuilder.append(profilerOption);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString());
  }

  Map<String, Object> myConfigMap = new HashMap<String, Object>(conf);
  StringWriter out = new StringWriter();

  try {
    JSONValue.writeJSONString(myConfigMap, out);
  } catch (IOException e) {
    System.out.println("Error in writing JSONString");
    e.printStackTrace();
    return;
  }

  Config config = new Config();
  config.putAll(Utils.readStormConfig());

  NimbusClient nc = NimbusClient.getConfiguredClient(config);
  String topologyName = stormTopo.getTopologyName();
  try {
    System.out.println("Submitting topology with name: "
        + topologyName);
    nc.getClient().submitTopology(topologyName, uploadedJarLocation,
        out.toString(), stormTopo.getStormBuilder().createTopology());
    System.out.println(topologyName + " is successfully submitted");

  } catch (AlreadyAliveException aae) {
    System.out.println("Fail to submit " + topologyName
        + "\nError message: " + aae.get_msg());
  } catch (InvalidTopologyException ite) {
    System.out.println("Invalid topology for " + topologyName);
    ite.printStackTrace();
  } catch (TException te) {
    System.out.println("Texception for " + topologyName);
    te.printStackTrace();
  }
}
 

void submit() throws AlreadyAliveException, InvalidTopologyException {
  StormSubmitter.submitTopology(topologyName, stormConfig, builder.createTopology());
}
 

void submitLocal(ILocalCluster cluster) throws AlreadyAliveException, InvalidTopologyException {
  stormConfig.setDebug(true);
  cluster.submitTopology(topologyName, stormConfig, builder.createTopology());
}
 

private void assertNotAlive() throws AlreadyAliveException {
  // only one topology is allowed to run. A topology is running if the topologyName is set.
  if (this.topologyName != null) {
    throw new AlreadyAliveException();
  }
}
 

@SuppressWarnings("rawtypes")
void submitTopology(String topologyName, Map conf, StormTopology topology) throws
    AlreadyAliveException, InvalidTopologyException;
 

public static void SetRemoteTopology() throws AlreadyAliveException,
        InvalidTopologyException, TopologyAssignException {

    TopologyBuilder builder = SetBuilder();

    StormSubmitter.submitTopology(topologyName, conf,
            builder.createTopology());

}
 

/**
 * Submits a topology to run on the cluster with a progress bar. A topology runs forever or until
 * explicitly killed.
 *
 *
 * @param name the name of the storm.
 * @param stormConf the topology-specific configuration. See {@link Config}.
 * @param topology the processing to execute.
 * @throws AlreadyAliveException if a topology with this name is already running
 * @throws InvalidTopologyException if an invalid topology was submitted
 */

public static void submitTopologyWithProgressBar(String name, Map stormConf, StormTopology topology) throws AlreadyAliveException, InvalidTopologyException {
    submitTopologyWithProgressBar(name, stormConf, topology, null);
}