org.apache.storm.generated.AuthorizationException Java Examples

public static void main(String[] args) {
    prepare();
    BlobStoreAPIWordCountTopology wc = new BlobStoreAPIWordCountTopology();
    try {
        File file = createFile(fileName);
        // Creating blob again before launching topology
        createBlobWithContent(key, store, file);

        // Blostore launch command with topology blobstore map
        // Here we are giving it a local name so that we can read from the file
        // bin/storm jar examples/storm-starter/storm-starter-topologies-0.11.0-SNAPSHOT.jar
        // org.apache.storm.starter.BlobStoreAPIWordCountTopology bl -c
        // topology.blobstore.map='{"key":{"localname":"blacklist.txt", "uncompress":"false"}}'
        wc.buildAndLaunchWordCountTopology(args);

        // Updating file few times every 5 seconds
        for (int i = 0; i < 10; i++) {
            updateBlobWithContent(key, store, updateFile(file));
            Utils.sleep(5000);
        }
    } catch (KeyAlreadyExistsException kae) {
        LOG.info("Key already exists {}", kae);
    } catch (AuthorizationException | KeyNotFoundException | IOException exp) {
        throw new RuntimeException(exp);
    }
}
 

public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException, AuthorizationException {
   TopologyBuilder builder = new TopologyBuilder();
   builder.setSpout(DATA_GENERATOR, new ASpout());
   builder.setBolt(DATA_CALCULATOR, new ABolt()).shuffleGrouping(DATA_GENERATOR);
   builder.setBolt(DATA_PRINTER, new DataPrinter()).shuffleGrouping(DATA_CALCULATOR).shuffleGrouping(DATA_GENERATOR);

   Config config = new Config();

   LocalCluster cluster = new LocalCluster();
   cluster.submitTopology(TOPOLOGY_NAME, config,
         builder.createTopology());

   Thread.sleep(100000);
   cluster.killTopology(TOPOLOGY_NAME);
   cluster.shutdown();
}
 

public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
 	//定义拓扑
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("word-reader" , new WordReader());
     builder.setBolt("word-normalizer" , new WordNormalizer()).shuffleGrouping("word-reader" );
     builder.setBolt("word-counter" , new WordCounter()).fieldsGrouping("word-normalizer" , new Fields("word"));
     StormTopology topology = builder.createTopology();
     //配置
     
     Config conf = new Config();
     String fileName ="words.txt" ;
     conf.put("fileName" , fileName );
     conf.setDebug(false);
 
      //运行拓扑
      System.out.println("开始...");
      if(args !=null&&args.length>0){ //有参数时，表示向集群提交作业，并把第一个参数当做topology名称
     	 System.out.println("远程模式");
          try {
	StormSubmitter.submitTopology(args[0], conf, topology);
} catch (AuthorizationException e) {
	e.printStackTrace();
}
    } else{//没有参数时，本地提交
      //启动本地模式
 	 System.out.println("本地模式");
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("Getting-Started-Topologie" , conf , topology );
      Thread.sleep(5000);
      //关闭本地集群
      cluster.shutdown();
    }
      System.out.println("结束");
    
 }
 

public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
 	//定义拓扑
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("word-reader" , new WordReader());
     builder.setBolt("word-normalizer" , new WordNormalizer()).shuffleGrouping("word-reader" );
     builder.setBolt("word-counter" , new WordCounter()).fieldsGrouping("word-normalizer" , new Fields("word"));
     StormTopology topology = builder.createTopology();
     //配置
     
     Config conf = new Config();
     String fileName ="words.txt" ;
     conf.put("fileName" , fileName );
     conf.setDebug(false);
 
      //运行拓扑
      System.out.println("开始...");
      if(args !=null&&args.length>0){ //有参数时，表示向集群提交作业，并把第一个参数当做topology名称
     	 System.out.println("远程模式");
          try {
	StormSubmitter.submitTopology(args[0], conf, topology);
} catch (AuthorizationException e) {
	e.printStackTrace();
}
    } else{//没有参数时，本地提交
      //启动本地模式
 	 System.out.println("本地模式");
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("Getting-Started-Topologie" , conf , topology );
      Thread.sleep(5000);
      //关闭本地集群
      cluster.shutdown();
    }
      System.out.println("结束");
    
 }
 

public static void main(String[] args)
        throws InvalidTopologyException, AuthorizationException, AlreadyAliveException, InterruptedException {
    ConfigurableApplicationContext context = SpringApplication.run(DemoApplication.class, args);
    AppMain appMain = context.getBean(AppMain.class);
    appMain.Laugher();
    SpringApplication.exit(context);
}
 

public static void runPirkTopology() throws PIRException
{
  // Set up Kafka parameters
  logger.info("Configuring Kafka.");
  String zkRoot = "/" + kafkaTopic + "_pirk_storm";
  BrokerHosts zkHosts = new ZkHosts(brokerZk);
  SpoutConfig kafkaConfig = new SpoutConfig(zkHosts, kafkaTopic, zkRoot, kafkaClientId);
  kafkaConfig.ignoreZkOffsets = forceFromStart;

  // Create conf
  logger.info("Retrieving Query and generating Storm conf.");
  Config conf = createStormConf();
  Query query = StormUtils.getQuery(useHdfs, hdfsUri, queryFile);
  conf.put(StormConstants.N_SQUARED_KEY, query.getNSquared().toString());
  conf.put(StormConstants.QUERY_INFO_KEY, query.getQueryInfo().toMap());

  // Configure this for different types of input data on Kafka.
  kafkaConfig.scheme = new SchemeAsMultiScheme(new PirkHashScheme(conf));

  // Create topology
  StormTopology topology = getPirkTopology(kafkaConfig);

  // Run topology
  logger.info("Submitting Pirk topology to Storm...");
  try
  {
    StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, topology);
  } catch (AlreadyAliveException | InvalidTopologyException | AuthorizationException e)
  {
    throw new PIRException(e);
  }

}
 

public static void main(String[] args)
		throws InterruptedException, InvalidTopologyException, AuthorizationException, AlreadyAliveException {

	String topologyName = "TSAS";// 元组名
	// Zookeeper主机地址，会自动选取其中一个
	ZkHosts zkHosts = new ZkHosts("192.168.230.128:2181,192.168.230.129:2181,192.168.230.131:2181");
	String topic = "trademx";
	String zkRoot = "/storm";// storm在Zookeeper上的根路径
	String id = "tsaPro";

	// 创建SpoutConfig对象
	SpoutConfig spontConfig = new SpoutConfig(zkHosts, topic, zkRoot, id);

	TopologyBuilder builder = new TopologyBuilder();
	builder.setSpout("kafka", new KafkaSpout(spontConfig), 2);
	builder.setBolt("AccBolt", new AccBolt()).shuffleGrouping("kafka");
	builder.setBolt("ToDbBolt", new ToDbBolt()).shuffleGrouping("AccBolt");

	Config config = new Config();
	config.setDebug(false);

	if (args.length == 0) { // 本地运行，用于测试
		LocalCluster localCluster = new LocalCluster();
		localCluster.submitTopology(topologyName, config, builder.createTopology());
		Thread.sleep(1000 * 3600);
		localCluster.killTopology(topologyName);
		localCluster.shutdown();
	} else { // 提交至集群运行
		StormSubmitter.submitTopology(topologyName, config, builder.createTopology());
	}

}
 

private static void createBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file)
    throws AuthorizationException, KeyAlreadyExistsException, IOException, KeyNotFoundException {
    String stringBlobACL = "o::rwa";
    AccessControl blobACL = BlobStoreAclHandler.parseAccessControl(stringBlobACL);
    List<AccessControl> acls = new LinkedList<AccessControl>();
    acls.add(blobACL); // more ACLs can be added here
    SettableBlobMeta settableBlobMeta = new SettableBlobMeta(acls);
    AtomicOutputStream blobStream = clientBlobStore.createBlob(blobKey, settableBlobMeta);
    blobStream.write(readFile(file).toString().getBytes());
    blobStream.close();
}
 

public void buildAndLaunchWordCountTopology(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomSentenceSpout(), 5);
    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("filter", new FilterWords(), 6).shuffleGrouping("split");

    Config conf = new Config();
    conf.setDebug(true);
    try {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    } catch (InvalidTopologyException | AuthorizationException | AlreadyAliveException exp) {
        throw new RuntimeException(exp);
    }
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
    Intialize(args[0]);
    logger.info("Successfully loaded Configuration ");


    BrokerHosts hosts = new ZkHosts(zkhost);
    SpoutConfig spoutConfig = new SpoutConfig(hosts, inputTopic, "/" + KafkaBroker, consumerGroup);
    spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
    spoutConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
    KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
    String[] partNames = {"status_code"};
    String[] colNames = {"date", "request_url", "protocol_type", "status_code"};

    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames))
            .withPartitionFields(new Fields(partNames));


    HiveOptions hiveOptions;
    //make sure you change batch size and all paramtere according to requirement
    hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(250).withBatchSize(2)
            .withIdleTimeout(10).withCallTimeout(10000000);

    logger.info("Creating Storm Topology");
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("KafkaSpout", kafkaSpout, 1);

    builder.setBolt("frauddetect", new FraudDetectorBolt()).shuffleGrouping("KafkaSpout");
    builder.setBolt("KafkaOutputBolt",
            new IPFraudKafkaBolt(zkhost, "kafka.serializer.StringEncoder", KafkaBroker, outputTopic), 1)
            .shuffleGrouping("frauddetect");

    builder.setBolt("HiveOutputBolt", new IPFraudHiveBolt(), 1).shuffleGrouping("frauddetect");
    builder.setBolt("HiveBolt", new HiveBolt(hiveOptions)).shuffleGrouping("HiveOutputBolt");

    Config conf = new Config();
    if (args != null && args.length > 1) {
        conf.setNumWorkers(3);
        logger.info("Submiting  topology to storm cluster");

        StormSubmitter.submitTopology(args[1], conf, builder.createTopology());
    } else {
        // Cap the maximum number of executors that can be spawned
        // for a component to 3
        conf.setMaxTaskParallelism(3);
        // LocalCluster is used to run locally
        LocalCluster cluster = new LocalCluster();
        logger.info("Submitting  topology to local cluster");
        cluster.submitTopology("KafkaLocal", conf, builder.createTopology());
        // sleep
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            logger.error("Exception ocuured" + e);
            cluster.killTopology("KafkaToplogy");
            logger.info("Shutting down cluster");
            cluster.shutdown();
        }
        cluster.shutdown();

    }

}
 

public void Laugher() throws InvalidTopologyException, AuthorizationException, AlreadyAliveException, InterruptedException {
        Config config = new Config();
        remoteSubmit(stormProps,topologyBuilder,config);
//        localSubmit(stormProps.getTopologyName(),topologyBuilder,config);
    }
 

private static void remoteSubmit(StormProps stormProps, TopologyBuilder builder, Config conf)
        throws InvalidTopologyException, AuthorizationException, AlreadyAliveException {
    conf.setNumWorkers(stormProps.getTopologyWorkers());
    conf.setMaxSpoutPending(stormProps.getTopologyMaxSpoutPending());
    StormSubmitter.submitTopology(stormProps.getTopologyName(), conf, builder.createTopology());
}
 

private static Nimbus.Client runOnStormCluster(Config conf, StormTopology topology) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
    Map<String, Object> clusterConf = Utils.readStormConfig();
    StormSubmitter.submitTopologyWithProgressBar(TOPO_NAME, conf, topology);
    return NimbusClient.getConfiguredClient(clusterConf).getClient();
}
 

private static void updateBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file)
    throws KeyNotFoundException, AuthorizationException, IOException {
    AtomicOutputStream blobOutputStream = clientBlobStore.updateBlob(blobKey);
    blobOutputStream.write(readFile(file).toString().getBytes());
    blobOutputStream.close();
}