backtype.storm.Config#registerSerialization

Source File: SwitchBoltIT.java From flowmix with Apache License 2.0

6 votes

@Test
public void test_timeDiffActivated_countEviction() throws InterruptedException {
    Flow flow = new FlowBuilder()
            .id("flow")
            .flowDefs()
            .stream("stream1")
            .stopGate().open(Policy.TIME_DELTA_LT, 1000).close(Policy.TIME, 5).evict(Policy.COUNT, 5).end()
            .endStream()   // send ALL results to stream2 and not to standard output
            .endDefs()
            .createFlow();

    StormTopology topology = buildTopology(flow, 50);
    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, topology);

    Thread.sleep(5000);

    assertEquals(5, MockSinkBolt.getEvents().size());
}

Source File: SwitchBoltIT.java From flowmix with Apache License 2.0

6 votes

@Test
public void test_timeDiffActivated_timeEviction() throws InterruptedException {
    Flow flow = new FlowBuilder()
            .id("flow")
            .flowDefs()
            .stream("stream1")
            .stopGate().open(Policy.TIME_DELTA_LT, 5).close(Policy.TIME, 1).evict(Policy.TIME, 1).end()
            .endStream()   // send ALL results to stream2 and not to standard output
            .endDefs()
            .createFlow();

    StormTopology topology = buildTopology(flow, 50);
    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, topology);

    Thread.sleep(7000);

    int size = MockSinkBolt.getEvents().size();
    System.out.println("SIZE: " + size);
    assertTrue(size >= 50 && size <= 65);
}

Source File: ExampleRunner.java From flowmix with Apache License 2.0

6 votes

public void run() {

    StormTopology topology = new FlowmixBuilder()
        .setFlowLoader(new SimpleFlowLoaderSpout(provider.getFlows(), 60000))
        .setEventsLoader(new MockEventGeneratorSpout(getMockEvents(), 10))
        .setOutputBolt(new PrinterBolt())
        .setParallelismHint(6)
      .create()
    .createTopology();

    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.setMaxSpoutPending(5000);
    conf.setDebug(false);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("example-topology", conf, topology);
  }

Source File: SelectorBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testSelection_basic() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
        .select().fields("key1", "key2").end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertTrue(MockSinkBolt.getEvents().size() > 0);

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("key1"));
    assertNotNull(event.get("key2"));
    assertNull(event.get("key3"));
    assertNull(event.get("key4"));
    assertNull(event.get("key5"));
  }
}

Source File: SelectorBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testSelection_fieldsDontExistDontReturn() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .select().fields("key7").end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}

Source File: FilterBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testFilter_nothingPasses() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key3", "val50").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}

Source File: FilterBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testFilter_eventsPass() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key1", "val1").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertTrue(MockSinkBolt.getEvents().size() > 0);

}

Source File: RichSpoutBatchTriggerer.java From jstorm with Apache License 2.0

5 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Map<String, Object> conf = _delegate.getComponentConfiguration();
    if(conf==null) conf = new HashMap<>();
    else conf = new HashMap<>(conf);
    Config.registerSerialization(conf, RichSpoutBatchId.class, RichSpoutBatchIdSerializer.class);
    return conf;
}

Source File: MasterBatchCoordinator.java From jstorm with Apache License 2.0

5 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}

Source File: AvroUtils.java From jstorm with Apache License 2.0

5 votes

/**
 * A helper method to extract avro serialization configurations from the topology configuration and register
 * specific kryo serializers as necessary.  A default serializer will be provided if none is specified in the
 * configuration.  "avro.serializer" should specify the complete class name of the serializer, e.g.
 * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer"
 *
 * @param conf The topology configuration
 * @throws ClassNotFoundException If the specified serializer cannot be located.
 */
public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException {
    final Class serializerClass;
    if (conf.containsKey("avro.serializer")) {
        serializerClass = Class.forName((String)conf.get("avro.serializer"));
    }
    else {
        serializerClass = GenericAvroSerializer.class;
    }
    conf.registerSerialization(GenericData.Record.class, serializerClass);
    conf.setSkipMissingKryoRegistrations(false);
}

Source File: MemoryTransactionalSpout.java From jstorm with Apache License 2.0

4 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
    return conf;
}

Source File: OpaqueMemoryTransactionalSpout.java From jstorm with Apache License 2.0

4 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
    return conf;
}

Source File: SequenceTopology.java From jstorm with Apache License 2.0

4 votes

public static void SetBuilder(TopologyBuilder builder, Map conf) {
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    
    if (!isEnableSplit) {
        BoltDeclarer boltDeclarer = builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
                bolt_Parallelism_hint);
                
        // localFirstGrouping is only for jstorm
        // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        boltDeclarer.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
                .allGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, SequenceTopologyDef.CONTROL_STREAM_ID)
                .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
    } else {
        
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint)
                .localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
                
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
                
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint)
                .fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID"))
                .fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
                
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint)
                .noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        
        Config.registerSerialization(conf, TradeCustomer.class, TradeCustomerSerializer.class);
        Config.registerSerialization(conf, Pair.class, PairSerializer.class);
    }
    
    // conf.put(Config.TOPOLOGY_DEBUG, false);
    // conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
    // conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    // conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
    // conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
    // conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);
    
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
    
}

Source File: SequenceTopologyTool.java From jstorm with Apache License 2.0

4 votes

public StormTopology buildTopology() {
    Config conf = getConf();
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    
    if (!isEnableSplit) {
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint)
                .localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    } else {
        
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint)
                .localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
                
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
                
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint)
                .fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID"))
                .fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
                
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint)
                .noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        
        Config.registerSerialization(conf, TradeCustomer.class);
        Config.registerSerialization(conf, Pair.class);
    }
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
    
    return builder.createTopology();
}

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_countTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 5)
          .evict(Policy.COUNT, 2)
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents().size());
  assertTrue(MockSinkBolt.getEvents().size() > 130);
  assertTrue(MockSinkBolt.getEvents().size() < 160);

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 2);
  }
}

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_countTrigger_timeEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 600)       // trigger every 500 events received
          .evict(Policy.TIME, 1)            // only keep last 1 second in the window
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }


  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(1, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() > 90);
    assertTrue(event.<Long>get("aCount").getValue() < 100);
  }
}

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_timeTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.TIME, 2)
          .evict(Policy.COUNT, 10)
          .clearOnTrigger()
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);


  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(4, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 10);
  }
}

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_timeTrigger_timeEvict() {

    Flow flow = new FlowBuilder()
        .id("myflow")
        .flowDefs()
            .stream("stream1")
                .partition().fields("key3").end()
                .aggregate().aggregator(CountAggregator.class)
                  .config(OUTPUT_FIELD, "aCount")
                  .trigger(Policy.TIME, 5)
                  .evict(Policy.TIME, 10)
                  .clearOnTrigger()
                .end()
            .endStream()
        .endDefs()
    .createFlow();

    StormTopology topology = buildTopology(flow, 10);
    Config conf = new Config();
    conf.registerSerialization(Event.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);
    conf.setNumWorkers(1);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(getTopologyName(), conf, topology);

    try {
      Thread.sleep(25000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }

    cluster.shutdown();

    System.out.println(MockSinkBolt.getEvents().size());
    assertEquals(4, MockSinkBolt.getEvents().size());

    for(Event event : MockSinkBolt.getEvents()) {
      assertNotNull(event.get("aCount"));
      assertTrue(event.<Long>get("aCount").getValue() > 350);
      assertTrue(event.<Long>get("aCount").getValue() < 500);
    }
}

Java Code Examples for backtype.storm.Config#registerSerialization()