backtype.storm.generated.InvalidTopologyException Java Examples

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public void submitTopology(String topoName,
                           Map config,
                           StormTopology stormTopology)
    throws AlreadyAliveException, InvalidTopologyException {
  assertNotAlive();

  this.topologyName = topoName;
  this.conf = config;
  this.topology = stormTopology;

  simulator.submitTopology(topoName,
      ConfigUtils.translateConfig(config),
      stormTopology.getStormTopology());
}
 

public void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    Config conf = getConf();
    StormTopology topology = buildTopology();
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    
    if (streamName.contains("zeromq")) {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.zeroMq.MQContext");
        
    } else {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.netty.NettyContext");
    }
    
    StormSubmitter.submitTopology(streamName, conf, topology);
    
}
 

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  if (args.length < 1) {
    throw new RuntimeException("Specify topology name");
  }

  int parallelism = 1;
  if (args.length > 1) {
    parallelism = Integer.parseInt(args[1]);
  }
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("word", new WordSpout(), parallelism);
  builder.setBolt("consumer", new ConsumerBolt(), parallelism)
      .fieldsGrouping("word", new Fields("word"));
  Config conf = new Config();
  conf.setNumWorkers(parallelism);

  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 

/**
 * generate TaskInfo for every bolt or spout in ZK /ZK/tasks/topoologyId/xxx
 */
public void setupZkTaskInfo(Map<Object, Object> conf, String topologyId, StormClusterState stormClusterState) throws Exception {
    Map<Integer, TaskInfo> taskToTaskInfo = mkTaskComponentAssignments(conf, topologyId);

    // mkdir /ZK/taskbeats/topoologyId
    int masterId = NimbusUtils.getTopologyMasterId(taskToTaskInfo);
    TopologyTaskHbInfo topoTaskHbInfo = new TopologyTaskHbInfo(topologyId, masterId);
    data.getTasksHeartbeat().put(topologyId, topoTaskHbInfo);
    stormClusterState.topology_heartbeat(topologyId, topoTaskHbInfo);

    if (taskToTaskInfo == null || taskToTaskInfo.size() == 0) {
        throw new InvalidTopologyException("Failed to generate TaskIDs map");
    }
    // key is task id, value is task info
    stormClusterState.set_task(topologyId, taskToTaskInfo);
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = JStormHelper.getConfig(args);
    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);
    boolean isValueSpout = JStormUtils.parseBoolean(conf.get("is.value.spout"), false);

    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isValueSpout)
        builder.setSpoutWithAck("spout", new BatchAckerValueSpout(), spoutParallelism);
    else
        builder.setSpoutWithAck("spout", new BatchAckerSpout(), spoutParallelism);
    builder.setBoltWithAck("split", new BatchAckerSplit(), splitParallelism).localOrShuffleGrouping("spout");;
    builder.setBoltWithAck("count", new BatchAckerCount(), countParallelism).fieldsGrouping("split", new Fields("word"));
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}
 

@Override
public void init(TopologyContext context) {
    try {
        this.topologyName = Common.topologyIdToName(context.getTopologyId());
    } catch (InvalidTopologyException e) {
        LOG.error("Failed get topology name by id-{}", context.getTopologyId());
        throw new RuntimeException(e.getMessage());
    }
    String workerDir = context.getWorkerIdDir();
    metricClient = new MetricClient(context);
    hdfsWriteLatency = metricClient.registerHistogram("HDFS write latency");
    hdfsDeleteLatency = metricClient.registerHistogram("HDFS delete latency");
    rocksDbFlushAndCpLatency = metricClient.registerHistogram("RocksDB flush and checkpoint latency");
    cleanPeriod = ConfigExtension.getTransactionBatchSnapshotTimeout(context.getStormConf()) * 5 * 1000;
    serializer = SerializerFactory.createSerailzer(context.getStormConf());
    initEnv(topologyName, context.getStormConf(), workerDir);
}
 

private static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  LOG.info("running benchmark " + name);
  IBenchmark benchmark =  (IBenchmark) Runner.getApplicationFromName(PACKAGE + "." + name);
  Config config = new Config();
  config.putAll(Utils.readStormConfig());
  config.setDebug(true);
  StormTopology topology = benchmark.getTopology(config);
  LocalCluster localCluster = new LocalCluster();
  localCluster.submitTopology(name, config, topology);
  final int runtime = BenchmarkUtils.getInt(config, MetricsCollectorConfig.METRICS_TOTAL_TIME,
          MetricsCollectorConfig.DEFAULT_TOTAL_TIME);
  IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
  collector.run();
  try {
    Thread.sleep(runtime);
  } catch (InterruptedException e) {
    LOG.error("benchmark interrupted", e);
  }
  localCluster.shutdown();
}
 

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }

    int parallelism = 10;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism)
            .fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
/*
Set config here
*/

    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 

public static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  if (name.startsWith("storm.benchmark.benchmarks")) {
    LOG.info("running benchmark " + name);
    runBenchmark((IBenchmark) getApplicationFromName(name));
  } else if (name.startsWith("storm.benchmark.tools.producer")) {
    LOG.info("running producer " + name);
    runProducer((IProducer) getApplicationFromName(name));
  } else {
    throw new RuntimeException(name + " is neither benchmark nor producer");
  }
}
 

public void SetDPRCTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    // LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
    // "exclamation");
    //
    // builder.addBolt(new TotalCount(), 3);
    //
    // Config conf = new Config();
    //
    // conf.setNumWorkers(3);
    // StormSubmitter.submitTopology("rpc", conf,
    // builder.createRemoteTopology());
    System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}
 

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    SetBuilder(builder, conf);
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
}
 

public static void SetDPRCTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    // LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
    // "exclamation");
    //
    // builder.addBolt(new TotalCount(), 3);
    //
    // Config conf = new Config();
    //
    // conf.setNumWorkers(3);
    // StormSubmitter.submitTopology("rpc", conf,
    // builder.createRemoteTopology());
    System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}
 

private static void runApplication(IApplication app)
        throws AlreadyAliveException, InvalidTopologyException {
  config.putAll(Utils.readStormConfig());
  String name = (String) config.get(Config.TOPOLOGY_NAME);
  topology = app.getTopology(config);
  StormSubmitter.submitTopology(name, config, topology);
}
 

public static void runBenchmark(IBenchmark benchmark)
        throws AlreadyAliveException, InvalidTopologyException,
        ClassNotFoundException, IllegalAccessException, InstantiationException {
  runApplication(benchmark);
  if (isMetricsEnabled()) {
    IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
    collector.run();
  }
}
 

public void rmTopology(String topologyId) {
    String topologyName;
    try {
        topologyName = Common.topologyIdToName(topologyId);
        setting.remove(topologyName);
        LOG.info("Remove {} netty metrics setting ", topologyName);
    } catch (InvalidTopologyException ignored) {
    }
}
 

private StormTopology createTopology()
        throws AlreadyAliveException, InvalidTopologyException {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    
    return builder.createTopology();
}
 

/**
 * @return Map[task id, component id]
 */
public Map<Integer, TaskInfo> mkTaskComponentAssignments(Map<Object, Object> conf, String topologyId)
        throws IOException, InvalidTopologyException, KeyNotFoundException {
    // we can directly pass stormConf from submit method ?
    Map<Object, Object> stormConf = StormConfig.read_nimbus_topology_conf(topologyId, data.getBlobStore());
    StormTopology rawTopology = StormConfig.read_nimbus_topology_code(topologyId, data.getBlobStore());
    StormTopology topology = Common.system_topology(stormConf, rawTopology);

    return Common.mkTaskInfo(stormConf, topology, topologyId);
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Map config = new Config();
    config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler");
    config.put(Config.TOPOLOGY_WORKERS, 2);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2);
    builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4);
    StormTopology topology = builder.createTopology();

    StormSubmitter.submitTopology("TMUdfTopology", config, topology);
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, UnknownHostException, InterruptedException {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	TopologyBuilder builder = new TopologyBuilder();
	BrokerHosts hosts = new ZkHosts(pt.getRequired("zookeeper"));
	SpoutConfig spoutConfig = new SpoutConfig(hosts, pt.getRequired("topic"), "/" + pt.getRequired("topic"), UUID.randomUUID().toString());
	spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
	KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
	builder.setSpout("source", kafkaSpout, pt.getInt("sourceParallelism"));

	builder.setBolt("sink", new Throughput.Sink(pt), pt.getInt("sinkParallelism")).noneGrouping("source");

	Config conf = new Config();
	conf.setDebug(false);

	if (!pt.has("local")) {
		conf.setNumWorkers(pt.getInt("par", 2));

		StormSubmitter.submitTopologyWithProgressBar("kafka-spout-"+pt.get("name", "no_name"), conf, builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(pt.getInt("par", 2));

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("kafka-spout", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}
}
 

public static void main(String[] args) throws IOException{
	Properties props = StormSamoaUtils.getProperties();
	
	String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
	if(uploadedJarLocation == null){
		logger.error("Invalid properties file. It must have key {}", 
				StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
		return;
	}
	
	List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
	int numWorkers = StormSamoaUtils.numWorkers(tmpArgs);
	
	args = tmpArgs.toArray(new String[0]);
	StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);

	Config conf = new Config();
	conf.putAll(Utils.readStormConfig());
	conf.putAll(Utils.readCommandLineOpts());
	conf.setDebug(false);
	conf.setNumWorkers(numWorkers);
	
	String profilerOption = 
			props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY);
	if(profilerOption != null){
		String topoWorkerChildOpts =  (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS);
		StringBuilder optionBuilder = new StringBuilder();
		if(topoWorkerChildOpts != null){
			optionBuilder.append(topoWorkerChildOpts);	
			optionBuilder.append(' ');
		}
		optionBuilder.append(profilerOption);
		conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString());
	}

	Map<String, Object> myConfigMap = new HashMap<String, Object>(conf);
	StringWriter out = new StringWriter();

	try {
		JSONValue.writeJSONString(myConfigMap, out);
	} catch (IOException e) {
		System.out.println("Error in writing JSONString");
		e.printStackTrace();
		return;
	}
	
	Config config = new Config();
	config.putAll(Utils.readStormConfig());
	
	String nimbusHost = (String) config.get(Config.NIMBUS_HOST);
			
	NimbusClient nc = new NimbusClient(nimbusHost);
	String topologyName = stormTopo.getTopologyName();
	try {
		System.out.println("Submitting topology with name: " 
				+ topologyName);
		nc.getClient().submitTopology(topologyName, uploadedJarLocation,
				out.toString(), stormTopo.getStormBuilder().createTopology());
		System.out.println(topologyName + " is successfully submitted");

	} catch (AlreadyAliveException aae) {
		System.out.println("Fail to submit " + topologyName
				+ "\nError message: " + aae.get_msg());
	} catch (InvalidTopologyException ite) {
		System.out.println("Invalid topology for " + topologyName);
		ite.printStackTrace();
	} catch (TException te) {
		System.out.println("Texception for " + topologyName);
		te.printStackTrace();
	} 		
}
 

public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
	
	TopologyRunner runner = new LancopeRunner();
	runner.initTopology(args, "lancope");
}
 

public SnapshotStateMaster(TopologyContext context, OutputCollector outputCollector) {
    this.topologyId = context.getTopologyId();
    try {
        this.topologyName = Common.topologyIdToName(topologyId);
    } catch (InvalidTopologyException e) {
        LOG.error("Failed to convert topologyId to topologyName", e);
        throw new RuntimeException(e);
    }
    this.topology = context.getRawTopology();
    this.conf = context.getStormConf();
    this.outputCollector = outputCollector;
    this.context = context;

    String topologyStateOpClassName = ConfigExtension.getTopologyStateOperatorClass(conf);
    if (topologyStateOpClassName == null) {
        stateOperator = new DefaultTopologyStateOperator();
    } else {
        stateOperator = (ITopologyStateOperator) Utils.newInstance(topologyStateOpClassName);
    }
    stateOperator.init(context);

    Set<String> spoutIds = topology.get_spouts().keySet();
    Set<String> statefulBoltIds = TransactionCommon.getStatefulBolts(topology);
    Set<String> endBolts = TransactionCommon.getEndBolts(topology);
    Set<String> downstreamComponents = new HashSet<>(topology.get_bolts().keySet());

    spouts = componentToComponentTasks(context, spoutIds);
    statefulBolts = componentToComponentTasks(context, statefulBoltIds);
    downstreamComponents.removeAll(statefulBoltIds);
    nonStatefulBoltTasks = componentToComponentTasks(context, downstreamComponents);
    endBoltTasks = new HashSet<Integer>(context.getComponentsTasks(endBolts));
    snapshotState = new SnapshotState(context, spouts, statefulBolts, nonStatefulBoltTasks, endBoltTasks, stateOperator);

    SnapshotState commitState = ConfigExtension.resetTransactionTopologyState(conf) ? null : (SnapshotState) stateOperator.initState(topologyName);
    snapshotState.initState(commitState);

    LOG.info("topologySnapshotState: {}, isResetTopologyState: {}", snapshotState, ConfigExtension.resetTransactionTopologyState(conf));
    LOG.info("lastSuccessfulSnapshotState: {}", snapshotState.getLastSuccessfulBatch().statesInfo());

    this.batchSnapshotTimeout = ConfigExtension.getTransactionBatchSnapshotTimeout(conf);
    scheduledService = Executors.newSingleThreadScheduledExecutor();
    scheduledService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            expiredCheck();
        }
    }, batchSnapshotTimeout, batchSnapshotTimeout / 2, TimeUnit.SECONDS);

    this.lock = new ReentrantLock(true);
}
 

@Override
public void validate(String topologyName, Map topologyConf, StormTopology topology) throws InvalidTopologyException {
}
 

void submitTopology(String topologyName, Map conf, StormTopology topology)
throws AlreadyAliveException, InvalidTopologyException;
 

void submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, SubmitOptions submitOpts)
throws AlreadyAliveException, InvalidTopologyException;
 

public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
	
	TopologyRunner runner = new PcapRunner();
	runner.initTopology(args, "pcap");
}
 

@SuppressWarnings("rawtypes")
void submitTopology(String topologyName, Map conf, StormTopology topology) throws
    AlreadyAliveException, InvalidTopologyException;
 

void submitLocal(ILocalCluster cluster) throws AlreadyAliveException, InvalidTopologyException {
  stormConfig.setDebug(true);
  cluster.submitTopology(topologyName, stormConfig, builder.createTopology());
}
 

void submit() throws AlreadyAliveException, InvalidTopologyException {
  StormSubmitter.submitTopology(topologyName, stormConfig, builder.createTopology());
}