org.apache.storm.tuple.Fields Java Examples

public static void main(String[] args) throws Exception {

        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout("spout", new RandomSentence(), 5);

        builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
        builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

        Config conf = new Config();
        conf.setDebug(true);
        String topoName = "word-count";
        if (args != null && args.length > 0) {
            topoName = args[0];
        }
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
    }
 

public static StormTopology buildTopology(WindowsStoreFactory windowsStore) throws Exception {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
                                                new Values("the man went to the store and bought some candy"),
                                                new Values("four score and seven years ago"),
                                                new Values("how many apples can you eat"), new Values("to be or not to be the person"));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();

    Stream stream = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
                                                                                 new Split(), new Fields("word"))
                            .window(TumblingCountWindow.of(1000), windowsStore, new Fields("word"), new CountAsAggregator(),
                                    new Fields("count"))
                            .peek(new Consumer() {
                                @Override
                                public void accept(TridentTuple input) {
                                    LOG.info("Received tuple: [{}]", input);
                                }
                            });

    return topology.build();
}
 

@SuppressWarnings("rawtypes")
public WorkerTopologyContext(
    StormTopology topology,
    Map stormConf,
    Map<Integer, String> taskToComponent,
    Map<String, List<Integer>> componentToSortedTasks,
    Map<String, Map<String, Fields>> componentToStreamToFields,
    String stormId,
    String codeDir,
    String pidDir,
    Integer workerPort,
    List<Integer> workerTasks,
    Map<String, Object> defaultResources,
    Map<String, Object> userResources
) {
  super(topology, stormConf, taskToComponent,
      componentToSortedTasks, componentToStreamToFields, stormId);
  throw new RuntimeException("WorkerTopologyContext should never be init this way");
}
 

@Test
public void testDeclaredOutputFields() {
    CustomOutputFieldsDeclarer declarer = new CustomOutputFieldsDeclarer();
    spout.declareOutputFields(declarer);
    Fields expected = new Fields(TopologyConstants.ID_FIELD, TopologyConstants.TICK_FIELD);
    Assert.assertTrue(declarer.areFieldsPresent(TopologyConstants.TICK_STREAM, false, expected));
}
 

@Override
protected int run(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();

    String[] testURLs = new String[] { "http://www.lequipe.fr/",
            "http://www.lemonde.fr/", "http://www.bbc.co.uk/",
            "http://storm.apache.org/", "http://digitalpebble.com/" };

    builder.setSpout("spout", new MemorySpout(testURLs));

    builder.setBolt("partitioner", new URLPartitionerBolt())
            .shuffleGrouping("spout");

    builder.setBolt("fetch", new FetcherBolt())
            .fieldsGrouping("partitioner", new Fields("key"));

    builder.setBolt("sitemap", new SiteMapParserBolt())
            .localOrShuffleGrouping("fetch");

    builder.setBolt("feeds", new FeedParserBolt())
            .localOrShuffleGrouping("sitemap");

    builder.setBolt("parse", new JSoupParserBolt())
            .localOrShuffleGrouping("feeds");

    builder.setBolt("index", new StdOutIndexer())
            .localOrShuffleGrouping("parse");

    Fields furl = new Fields("url");

    // can also use MemoryStatusUpdater for simple recursive crawls
    builder.setBolt("status", new StdOutStatusUpdater())
            .fieldsGrouping("fetch", Constants.StatusStreamName, furl)
            .fieldsGrouping("sitemap", Constants.StatusStreamName, furl)
            .fieldsGrouping("feeds", Constants.StatusStreamName, furl)
            .fieldsGrouping("parse", Constants.StatusStreamName, furl)
            .fieldsGrouping("index", Constants.StatusStreamName, furl);

    return submit("crawl", conf, builder);
}
 

public static void main(String[] args)  {
		//定义一个拓扑
		TopologyBuilder builder=new TopologyBuilder();
		//设置两个Executeor(线程)，默认一个
		builder.setSpout(test_spout, new TestSpout(),2);
		//shuffleGrouping:表示是随机分组
		//设置两个Executeor(线程)，和两个task
		builder.setBolt(test_bolt, new TestBolt(),2).setNumTasks(2).shuffleGrouping(test_spout);
		//fieldsGrouping:表示是按字段分组
		//设置两个Executeor(线程)，和两个task
		builder.setBolt(test2_bolt, new Test2Bolt(),2).setNumTasks(2).fieldsGrouping(test_bolt, new Fields("count"));
		Config conf = new Config();
		conf.put("test", "test");
		try{
		  //运行拓扑
	       if(args !=null&&args.length>0){ //有参数时，表示向集群提交作业，并把第一个参数当做topology名称
	       	 System.out.println("运行远程模式");
			 StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
	      } else{//没有参数时，本地提交
	        //启动本地模式
	     	System.out.println("运行本地模式");
	        LocalCluster cluster = new LocalCluster();
	        cluster.submitTopology("Word-counts" ,conf,  builder.createTopology() );
	        Thread.sleep(20000);
//	        //关闭本地集群
	        cluster.shutdown();
	      }
		}catch (Exception e){
			e.printStackTrace();
		}
	}
 

public static StormTopology buildTopology() {
    TridentTopology topology = new TridentTopology();
    TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB));
    TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB));


    topology.newDRPCStream("reach").stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields(
        "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle().stateQuery(
        tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"),
                                                                                                new ExpandList(),
                                                                                                new Fields("follower"))
            .groupBy(new Fields("follower")).aggregate(new One(), new Fields(
        "one")).aggregate(new Fields("one"), new Sum(), new Fields("reach"));
    return topology.build();
}
 

@BeforeMethod
public void beforeMethod() throws Exception {
  // Fixtures
  alarmDefinitionDAO = mock(AlarmDefinitionDAO.class);

  // Mocks
  alarmDAO = new MockAlarmDAO();

  // Bindings
  Injector.reset();
  Injector.registerModules(new AbstractModule() {
    protected void configure() {
      bind(AlarmDAO.class).toInstance(alarmDAO);
      bind(AlarmDefinitionDAO.class).toInstance(alarmDefinitionDAO);
    }
  });

  // Config
  ThresholdingConfiguration threshConfig = new ThresholdingConfiguration();
  threshConfig.alarmDelay = 1;
  threshConfig.sporadicMetricNamespaces = new HashSet<String>();
  Serialization.registerTarget(KafkaProducerConfiguration.class);

  threshConfig.kafkaProducerConfig =
      Serialization
          .fromJson("{\"KafkaProducerConfiguration\":{\"topic\":\"alarm-state-transitions\",\"metadataBrokerList\":\"192.168.10.10:9092\",\"requestRequiredAcks\":1,\"requestTimeoutMs\":10000,\"producerType\":\"sync\",\"serializerClass\":\"kafka.serializer.StringEncoder\",\"keySerializerClass\":\"\",\"partitionerClass\":\"\",\"compressionCodec\":\"none\",\"compressedTopics\":\"\",\"messageSendMaxRetries\":3,\"retryBackoffMs\":100,\"topicMetadataRefreshIntervalMs\":600000,\"queueBufferingMaxMs\":5000,\"queueBufferingMaxMessages\":10000,\"queueEnqueueTimeoutMs\":-1,\"batchNumMessages\":200,\"sendBufferBytes\":102400,\"clientId\":\"Threshold_Engine\"}}");
  Config stormConfig = new Config();
  stormConfig.setMaxTaskParallelism(1);
  metricSpout = new FeederSpout(new Fields(MetricSpout.FIELDS));
  eventSpout = new FeederSpout(new Fields("event"));
  alarmEventForwarder = mock(AlarmEventForwarder.class);
  Injector
      .registerModules(new TopologyModule(threshConfig, stormConfig, metricSpout, eventSpout));
  Injector.registerModules(new ProducerModule(alarmEventForwarder));

  // Evaluate alarm stats every 5 seconds
  System.setProperty(MetricAggregationBolt.TICK_TUPLE_SECONDS_KEY, "5");

  startTopology();
}
 

@SuppressWarnings("rawtypes")
public GeneralTopologyContext(StormTopology topology, Map stormConf,
                              Map<Integer, String> taskToComponent,
                              Map<String, List<Integer>> componentToSortedTasks,
                              Map<String, Map<String, Fields>> componentToStreamToFields,
                              String stormId) {
  throw new RuntimeException("GeneralTopologyContext should never be initiated this way");
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    if (!consoleNotificationStream.isEmpty()) {
        declarer.declareStream(consoleNotificationStream, new Fields(StreamlineEvent.STREAMLINE_EVENT));
    }
    if (!emailNotificationStream.isEmpty()) {
        declarer.declareStream(emailNotificationStream, new Fields(StreamlineEvent.STREAMLINE_EVENT));
    }
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    if (normalizationProcessor == null) {
        throw new RuntimeException("normalizationProcessor cannot be null");
    }
    for (Stream stream : normalizationProcessor.getOutputStreams()) {
        declarer.declareStream(stream.getId(), new Fields(StreamlineEvent.STREAMLINE_EVENT));
    }
}
 

/**
 * Only output the specified fields.
 *
 * @param commaSeparatedFields  Names of output fields, in the order they should appear in file
 * @return
 */
public HdfsTextOutputFormat withFields(String commaSeparatedFields){
    String[] rawFields = commaSeparatedFields.split(",");
    String[] trimmedFields = new String[rawFields.length];
    for (int i = 0; i < rawFields.length; i++)   {
        trimmedFields[i] = rawFields[i].trim();
    }

    fields = new Fields(trimmedFields);
    return this;
}
 

@Test
public void oneArgument() {
	SpringRPCRequest subject = new SpringRPCRequest("fn(x)");
	OutputFieldsDeclarer declarerMock = mock(OutputFieldsDeclarer.class);
	subject.declareOutputFields(declarerMock);
	ArgumentCaptor<Fields> fieldsCaptor = ArgumentCaptor.forClass(Fields.class);
	verify(declarerMock).declareStream(eq("default"), fieldsCaptor.capture());
	assertEquals(asList("x", "fn-rpc-ctx"), fieldsCaptor.getValue().toList());
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
    final List<String> fieldNames = new ArrayList<>();
    fieldNames.add(NIFI_DATA_PACKET);
    fieldNames.addAll(attributeNames);
    outputFieldsDeclarer.declare(new Fields(fieldNames));
}
 

/**
 * Creates a topology which demonstrates min/max operations on tuples of stream which contain vehicle and driver fields
 * with values {@link TridentMinMaxOfVehiclesTopology.Vehicle} and {@link TridentMinMaxOfVehiclesTopology.Driver} respectively.
 */
public static StormTopology buildVehiclesTopology() {
    Fields driverField = new Fields(Driver.FIELD_NAME);
    Fields vehicleField = new Fields(Vehicle.FIELD_NAME);
    Fields allFields = new Fields(Vehicle.FIELD_NAME, Driver.FIELD_NAME);

    FixedBatchSpout spout = new FixedBatchSpout(allFields, 10, Vehicle.generateVehicles(20));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream vehiclesStream = topology.newStream("spout1", spout).
        each(allFields, new Debug("##### vehicles"));

    Stream slowVehiclesStream =
        vehiclesStream
            .min(new SpeedComparator())
            .each(vehicleField, new Debug("#### slowest vehicle"));

    Stream slowDriversStream =
        slowVehiclesStream
            .project(driverField)
            .each(driverField, new Debug("##### slowest driver"));

    vehiclesStream
        .max(new SpeedComparator())
        .each(vehicleField, new Debug("#### fastest vehicle"))
        .project(driverField)
        .each(driverField, new Debug("##### fastest driver"));

    vehiclesStream
        .minBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).
        each(vehicleField, new Debug("#### least efficient vehicle"));

    vehiclesStream
        .maxBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).
        each(vehicleField, new Debug("#### most efficient vehicle"));

    return topology.build();
}
 

@Test
public void shouldDeclareOutputFields() {
    // given
    OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
    IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();

    // when
    bolt.declareOutputFields(declarer);

    // then
    verify(declarer, times(1)).declare(any(Fields.class));
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declareStream(QUERY_STREAM, new Fields(ID_FIELD, QUERY_FIELD, METADATA_FIELD));
    declarer.declareStream(METADATA_STREAM, new Fields(ID_FIELD, METADATA_FIELD));
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
  outputFieldsDeclarer.declare(new Fields("sentence"));
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
  declarer.declare(new Fields("word"));
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
  declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
  outputFieldsDeclarer.declare(new Fields("sentence"));
}
 

public Fields getComponentOutputFields(String componentId, String streamId) {
    return fields;
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declare(new Fields("subPackage", "currentOffset"));
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declare(new Fields("record"));
}
 

public void addInboundFieldsForEdge(String edge, Fields fields) {
  LOG.info(String.format("[Storm-Bolt : %s] Adding inbound fields for edge %s "
      + "with fields %s", id, edge, fields));
  this.inboundEdgeToFieldsMap.put(edge, fields);
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
  declarer.declare(new Fields("ibasic-print"));
}
 

private void checkEnvelopeMetadata(RawMessage m) {
  assertEquals("real_original_string", m.getMetadata().get(MetadataUtil.METADATA_PREFIX + "." + Constants.Fields.ORIGINAL.getName()));
  assertEquals("enveloped_metadata_val_1", m.getMetadata().get(MetadataUtil.METADATA_PREFIX + ".enveloped_metadata_field_1"));
  assertEquals("enveloped_metadata_val_2", m.getMetadata().get(MetadataUtil.METADATA_PREFIX + ".enveloped_metadata_field_2"));
}
 

@Override
public Fields getOutputFields() {
    if(outputFields.size() == 0) return null;
    return new Fields(outputFields);
}
 

public void declareStream(String streamId, boolean direct, Fields fields) {
  delegate.declareStream(streamId, direct, fields.getDelegate());
}
 

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declare(new Fields("url", "content", "metadata", "text"));
    declarer.declareStream("tika", new Fields("url", "content", "metadata",
            "text"));
}