org.apache.storm.testing.FeederSpout Java Examples

The following examples show how to use org.apache.storm.testing.FeederSpout. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RTJoinExampleTopology.java    From streamline with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
//        if (!NimbusClient.isLocalOverride()) {
//            throw new IllegalStateException("This example only works in local mode.  "
//                    + "Run with storm local not storm jar");
//        }
        FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
        FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("genderSpout", genderSpout);
        builder.setSpout("ageSpout", ageSpout);

        // inner join of 'age' and 'gender' records on 'id' field
        RealtimeJoinBolt joiner = new RealtimeJoinBolt(StreamKind.SOURCE)
                .select("genderSpout:id,ageSpout:id,gender,age")
                .from("genderSpout", 5, false)
                .outerJoin("ageSpout", Duration.ofSeconds(5), false, Cmp.equal("genderSpout:id", "ageSpout:id") )
                .withOutputStream("jstream");

        builder.setBolt("joiner", joiner)
                .fieldsGrouping("genderSpout", new Fields("id"))
                .fieldsGrouping("ageSpout", new Fields("id"))         ;

        builder.setBolt("printer", new PrinterBolt() ).shuffleGrouping("joiner", "jstream");

        Config conf = new Config();
        StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology());

        generateGenderData(genderSpout);

        generateAgeData(ageSpout);
    }
Example #2
Source File: RTJoinExampleTopology.java    From streamline with Apache License 2.0 5 votes vote down vote up 
private static void generateGenderData(FeederSpout genderSpout) {
    for (int i = 0; i < 10; i++) {
        String gender;
        if (i % 2 == 0) {
            gender = "male";
        }
        else {
            gender = "female";
        }
        genderSpout.feed(new Values(i, gender));
    }
}
Example #3
Source File: SingleJoinExample.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
    if (!NimbusClient.isLocalOverride()) {
        throw new IllegalStateException("This example only works in local mode.  "
                                        + "Run with storm local not storm jar");
    }
    FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
    FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("gender", genderSpout);
    builder.setSpout("age", ageSpout);
    builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id"))
           .fieldsGrouping("age", new Fields("id"));

    Config conf = new Config();
    conf.setDebug(true);
    StormSubmitter.submitTopology("join-example", conf, builder.createTopology());

    for (int i = 0; i < 10; i++) {
        String gender;
        if (i % 2 == 0) {
            gender = "male";
        } else {
            gender = "female";
        }
        genderSpout.feed(new Values(i, gender));
    }

    for (int i = 9; i >= 0; i--) {
        ageSpout.feed(new Values(i, i + 20));
    }
}
Example #4
Source File: JoinBoltExample.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
    if (!NimbusClient.isLocalOverride()) {
        throw new IllegalStateException("This example only works in local mode.  "
                                        + "Run with storm local not storm jar");
    }
    FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
    FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("genderSpout", genderSpout);
    builder.setSpout("ageSpout", ageSpout);

    // inner join of 'age' and 'gender' records on 'id' field
    JoinBolt joiner = new JoinBolt("genderSpout", "id")
        .join("ageSpout", "id", "genderSpout")
        .select("genderSpout:id,ageSpout:id,gender,age")
        .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS));

    builder.setBolt("joiner", joiner)
           .fieldsGrouping("genderSpout", new Fields("id"))
           .fieldsGrouping("ageSpout", new Fields("id"));

    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("joiner");

    Config conf = new Config();
    StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology());

    generateGenderData(genderSpout);

    generateAgeData(ageSpout);
}
Example #5
Source File: JoinBoltExample.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up 
private static void generateGenderData(FeederSpout genderSpout) {
    for (int i = 0; i < 10; i++) {
        String gender;
        if (i % 2 == 0) {
            gender = "male";
        } else {
            gender = "female";
        }
        genderSpout.feed(new Values(i, gender));
    }
}
Example #6
Source File: ThresholdingEngineAlarmTest.java    From monasca-thresh with Apache License 2.0 5 votes vote down vote up 
@BeforeMethod
public void beforeMethod() throws Exception {
  // Fixtures
  alarmDefinitionDAO = mock(AlarmDefinitionDAO.class);

  // Mocks
  alarmDAO = new MockAlarmDAO();

  // Bindings
  Injector.reset();
  Injector.registerModules(new AbstractModule() {
    protected void configure() {
      bind(AlarmDAO.class).toInstance(alarmDAO);
      bind(AlarmDefinitionDAO.class).toInstance(alarmDefinitionDAO);
    }
  });

  // Config
  ThresholdingConfiguration threshConfig = new ThresholdingConfiguration();
  threshConfig.alarmDelay = 1;
  threshConfig.sporadicMetricNamespaces = new HashSet<String>();
  Serialization.registerTarget(KafkaProducerConfiguration.class);

  threshConfig.kafkaProducerConfig =
      Serialization
          .fromJson("{\"KafkaProducerConfiguration\":{\"topic\":\"alarm-state-transitions\",\"metadataBrokerList\":\"192.168.10.10:9092\",\"requestRequiredAcks\":1,\"requestTimeoutMs\":10000,\"producerType\":\"sync\",\"serializerClass\":\"kafka.serializer.StringEncoder\",\"keySerializerClass\":\"\",\"partitionerClass\":\"\",\"compressionCodec\":\"none\",\"compressedTopics\":\"\",\"messageSendMaxRetries\":3,\"retryBackoffMs\":100,\"topicMetadataRefreshIntervalMs\":600000,\"queueBufferingMaxMs\":5000,\"queueBufferingMaxMessages\":10000,\"queueEnqueueTimeoutMs\":-1,\"batchNumMessages\":200,\"sendBufferBytes\":102400,\"clientId\":\"Threshold_Engine\"}}");
  Config stormConfig = new Config();
  stormConfig.setMaxTaskParallelism(1);
  metricSpout = new FeederSpout(new Fields(MetricSpout.FIELDS));
  eventSpout = new FeederSpout(new Fields("event"));
  alarmEventForwarder = mock(AlarmEventForwarder.class);
  Injector
      .registerModules(new TopologyModule(threshConfig, stormConfig, metricSpout, eventSpout));
  Injector.registerModules(new ProducerModule(alarmEventForwarder));

  // Evaluate alarm stats every 5 seconds
  System.setProperty(MetricAggregationBolt.TICK_TUPLE_SECONDS_KEY, "5");

  startTopology();
}
Example #7
Source File: RTJoinExampleTopology.java    From streamline with Apache License 2.0 4 votes vote down vote up 
private static void generateAgeData(FeederSpout ageSpout) {
    for (int i = 9; i >= 0; i--) {
        ageSpout.feed(new Values(i, i + 20));
    }
}
Example #8
Source File: JoinBoltExample.java    From storm-net-adapter with Apache License 2.0 4 votes vote down vote up 
private static void generateAgeData(FeederSpout ageSpout) {
    for (int i = 9; i >= 0; i--) {
        ageSpout.feed(new Values(i, i + 20));
    }
}
Example #9
Source File: ThresholdingEngineTest.java    From monasca-thresh with Apache License 2.0 4 votes vote down vote up 
@BeforeMethod
public void befortMethod() throws Exception {
  // Fixtures
  final AlarmExpression expression =
      new AlarmExpression("max(cpu{id=5}) >= 3 or max(mem{id=5}) >= 5");
  final AlarmExpression expression2 = AlarmExpression.of(
    "count(log.error{id=5},deterministic) >= 1 OR count(log.warning{id=5},deterministic) >= 1"
  );
  final AlarmExpression expression3 = AlarmExpression.of(
    "max(cpu{id=5}) >= 3 AND count(log.warning{id=5},deterministic) >= 1"
  );

  cpuMetricDef = expression.getSubExpressions().get(0).getMetricDefinition();
  memMetricDef = expression.getSubExpressions().get(1).getMetricDefinition();
  logErrorMetricDef = expression2.getSubExpressions().get(0).getMetricDefinition();
  logWarningMetricDef = expression2.getSubExpressions().get(1).getMetricDefinition();

  extraMemMetricDefDimensions = new HashMap<>(memMetricDef.dimensions);
  extraMemMetricDefDimensions.put("Group", "group A");

  alarmDefinition =
      new AlarmDefinition(TEST_ALARM_TENANT_ID, TEST_ALARM_NAME,
          TEST_ALARM_DESCRIPTION, expression, "LOW", true, new ArrayList<String>());
  this.deterministicAlarmDefinition = new AlarmDefinition(
    DET_TEST_ALARM_TENANT_ID,
    DET_TEST_ALARM_NAME,
    DET_TEST_ALARM_DESCRIPTION,
    expression2,
    "LOW",
    true,
    new ArrayList<String>()
  );
  this.mixedAlarmDefinition = new AlarmDefinition(
    MIXED_TEST_ALARM_TENANT_ID,
    MIXED_TEST_ALARM_NAME,
    MIXED_TEST_ALARM_DESCRIPTION,
    expression3,
    "LOW",
    true,
    new ArrayList<String>()
  );

  // Mocks
  alarmDAO = mock(AlarmDAO.class);
  alarmDefinitionDAO = mock(AlarmDefinitionDAO.class);

  // Bindings
  Injector.reset();
  Injector.registerModules(new AbstractModule() {
    protected void configure() {
      bind(AlarmDAO.class).toInstance(alarmDAO);
      bind(AlarmDefinitionDAO.class).toInstance(alarmDefinitionDAO);
    }
  });

  // Config
  ThresholdingConfiguration threshConfig = new ThresholdingConfiguration();
  threshConfig.alarmDelay = 1;
  threshConfig.sporadicMetricNamespaces = new HashSet<String>();
  Serialization.registerTarget(KafkaProducerConfiguration.class);

  threshConfig.kafkaProducerConfig =
      Serialization
          .fromJson("{\"KafkaProducerConfiguration\":{\"topic\":\"alarm-state-transitions\",\"metadataBrokerList\":\"192.168.10.10:9092\",\"requestRequiredAcks\":1,\"requestTimeoutMs\":10000,\"producerType\":\"sync\",\"serializerClass\":\"kafka.serializer.StringEncoder\",\"keySerializerClass\":\"\",\"partitionerClass\":\"\",\"compressionCodec\":\"none\",\"compressedTopics\":\"\",\"messageSendMaxRetries\":3,\"retryBackoffMs\":100,\"topicMetadataRefreshIntervalMs\":600000,\"queueBufferingMaxMs\":5000,\"queueBufferingMaxMessages\":10000,\"queueEnqueueTimeoutMs\":-1,\"batchNumMessages\":200,\"sendBufferBytes\":102400,\"clientId\":\"Threshold_Engine\"}}");
  Config stormConfig = new Config();
  stormConfig.setMaxTaskParallelism(1);
  metricSpout = new FeederSpout(new Fields(MetricSpout.FIELDS));
  eventSpout = new FeederSpout(new Fields("event"));
  alarmEventForwarder = mock(AlarmEventForwarder.class);
  Injector
      .registerModules(new TopologyModule(threshConfig, stormConfig, metricSpout, eventSpout));
  Injector.registerModules(new ProducerModule(alarmEventForwarder));

}