storm.trident.state.StateFactory Java Examples

public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    StateFactory clickThruMemory = new MemoryMapState.Factory();
    ClickThruSpout spout = new ClickThruSpout();
    Stream inputStream = topology.newStream("clithru", spout);
    TridentState clickThruState = inputStream.each(new Fields("username", "campaign", "product", "click"), new Filter("click", "true"))
            .each(new Fields("username", "campaign", "product", "click"), new Distinct())
            .groupBy(new Fields("campaign"))
            .persistentAggregate(clickThruMemory, new Count(), new Fields("click_thru_count"));

    inputStream.groupBy(new Fields("campaign"))
            .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("impression_count"))
            .newValuesStream()
            .stateQuery(clickThruState, new Fields("campaign"), new MapGet(), new Fields("click_thru_count"))
            .each(new Fields("campaign", "impression_count", "click_thru_count"), new CampaignEffectiveness(), new Fields(""));

    return topology.build();
}
 

public static StormTopology buildTopology(String hdfsUrl){
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields hdfsFields = new Fields("sentence", "key");

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/trident")
            .withPrefix("trident")
            .withExtension(".txt");

    RecordFormat recordFormat = new DelimitedRecordFormat()
            .withFields(hdfsFields);

    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    HdfsState.Options options = new HdfsState.HdfsFileOptions()
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(recordFormat)
            .withRotationPolicy(rotationPolicy)
            .withFsUrl(hdfsUrl);

    StateFactory factory = new HdfsStateFactory().withOptions(options);

    TridentState state = stream
            .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

    return topology.build();
}
 

public static StormTopology buildTopology(String hdfsUrl){
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields hdfsFields = new Fields("sentence", "key");

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/trident")
            .withPrefix("trident")
            .withExtension(".seq");

    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions()
            .withFileNameFormat(fileNameFormat)
            .withSequenceFormat(new DefaultSequenceFormat("key", "sentence"))
            .withRotationPolicy(rotationPolicy)
            .withFsUrl(hdfsUrl)
            .addRotationAction(new MoveFileAction().toDestination("/dest2/"));

    StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);

    TridentState state = stream
            .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

    return topology.build();
}
 

private Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator,
                      Fields functionFields, boolean storeTuplesInStore) {
    projectionValidation(inputFields);
    windowConfig.validate();

    Fields fields = addTriggerField(functionFields);

    // when storeTuplesInStore is false then the given windowStoreFactory is only used to store triggers and
    // that store is passed to WindowStateUpdater to remove them after committing the batch.
    Stream stream = _topology.addSourcedNode(this,
            new ProcessorNode(_topology.getUniqueStreamId(),
                    _name,
                    fields,
                    fields,
                    new WindowTridentProcessor(windowConfig, _topology.getUniqueWindowId(), windowStoreFactory,
                            inputFields, aggregator, storeTuplesInStore)));

    Stream effectiveStream = stream.project(functionFields);

    // create StateUpdater with the given windowStoreFactory to remove triggered aggregation results form store
    // when they are successfully processed.
    StateFactory stateFactory = new WindowsStateFactory();
    StateUpdater stateUpdater = new WindowsStateUpdater(windowStoreFactory);
    stream.partitionPersist(stateFactory, new Fields(WindowTridentProcessor.TRIGGER_FIELD_NAME), stateUpdater, new Fields());

    return effectiveStream;
}
 

public TridentState partitionPersist(StateFactory stateFactory, Fields inputFields, StateUpdater updater) {
    return partitionPersist(stateFactory, inputFields, updater, new Fields());
}
 

public TridentState newStaticState(StateFactory factory) {
    return newStaticState(new StateSpec(factory));
}
 

public TridentState persistentAggregate(StateFactory stateFactory, ReducerAggregator agg, Fields functionFields) {
    return persistentAggregate(new StateSpec(stateFactory), agg, functionFields);
}
 

public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, ReducerAggregator agg, Fields functionFields) {
    return persistentAggregate(new StateSpec(stateFactory), inputFields, agg, functionFields);
}
 

public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, CombinerAggregator agg, Fields functionFields) {
    return persistentAggregate(new StateSpec(stateFactory), inputFields, agg, functionFields);
}
 

public TridentState persistentAggregate(StateFactory stateFactory, CombinerAggregator agg, Fields functionFields) {
    return persistentAggregate(new StateSpec(stateFactory), agg, functionFields);
}
 

public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, ReducerAggregator agg, Fields functionFields) {
    return persistentAggregate(new StateSpec(stateFactory), inputFields, agg, functionFields);
}
 

public TridentState persistentAggregate(StateFactory stateFactory, ReducerAggregator agg, Fields functionFields) {
    return persistentAggregate(new StateSpec(stateFactory), agg, functionFields);
}
 

public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, CombinerAggregator agg, Fields functionFields) {
    return persistentAggregate(new StateSpec(stateFactory), inputFields, agg, functionFields);
}
 

public TridentState persistentAggregate(StateFactory stateFactory, CombinerAggregator agg, Fields functionFields) {
    return persistentAggregate(new StateSpec(stateFactory), agg, functionFields);
}
 

public TridentState partitionPersist(StateFactory stateFactory, StateUpdater updater) {
    return partitionPersist(stateFactory, updater, new Fields());
}
 

public TridentState partitionPersist(StateFactory stateFactory, StateUpdater updater, Fields functionFields) {
    return partitionPersist(new StateSpec(stateFactory), updater, functionFields);
}
 

public TridentState partitionPersist(StateFactory stateFactory, Fields inputFields, StateUpdater updater, Fields functionFields) {
  return partitionPersist(new StateSpec(stateFactory), inputFields, updater, functionFields);
}
 

private static StormTopology externalState(LocalDRPC drpc, FeederBatchSpout spout) {
    TridentTopology topology = new TridentTopology();

    // You can reference existing data sources as well.
    // Here we are mocking up a "database"
    StateFactory stateFactory = new StateFactory() {
        @Override
        public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
            MemoryMapState<Integer> name_to_age = new MemoryMapState<Integer>("name_to_age");
            // This is a bit hard to read but it's just pre-populating the state
            List<List<Object>> keys = getKeys("ted", "mary", "jason", "tom", "chuck");
            name_to_age.multiPut(keys, ImmutableList.of(32, 21, 45, 52, 18));
            return name_to_age;
        }
    };
    TridentState nameToAge =
            topology.newStaticState(stateFactory);

    // Let's setup another state that keeps track of actor's appearance counts per location
    TridentState countState =
            topology
                    .newStream("spout", spout)
                    .groupBy(new Fields("actor","location"))
                    .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"));

    // Now, let's calculate the average age of actors seen
    topology
            .newDRPCStream("age_stats", drpc)
            .stateQuery(countState, new TupleCollectionGet(), new Fields("actor", "location"))
            .stateQuery(nameToAge, new Fields("actor"), new MapGet(), new Fields("age"))
            .each(new Fields("actor","location","age"), new Print())
            .groupBy(new Fields("location"))
            .chainedAgg()
            .aggregate(new Count(), new Fields("count"))
            .aggregate(new Fields("age"), new Sum(), new Fields("sum"))
            .chainEnd()
            .each(new Fields("sum", "count"), new DivideAsDouble(), new Fields("avg"))
            .project(new Fields("location", "count", "avg"))
    ;

    return topology.build();
}
 

@SuppressWarnings("rawtypes")
public static StateFactory nonTransactional(CqlRowMapper mapper, Options<Object> opts) {
    return new CassandraCqlMapStateFactory(mapper, StateType.NON_TRANSACTIONAL, opts);
}
 

@SuppressWarnings("rawtypes")
public static StateFactory nonTransactional(CqlRowMapper mapper) {
    Options<Object> options = new Options<Object>();
    return nonTransactional(mapper, options);
}
 

@SuppressWarnings("rawtypes")
public static StateFactory transactional(CqlRowMapper mapper, Options<TransactionalValue> opts) {
    return new CassandraCqlMapStateFactory(mapper, StateType.TRANSACTIONAL, opts);
}
 

@SuppressWarnings("rawtypes")
public static StateFactory transactional(CqlRowMapper mapper) {
    Options<TransactionalValue> options = new Options<TransactionalValue>();
    return transactional(mapper, options);
}
 

@SuppressWarnings("rawtypes")
public static StateFactory opaque(CqlRowMapper mapper, Options<OpaqueValue> opts) {
    return new CassandraCqlMapStateFactory(mapper, StateType.OPAQUE, opts);
}
 

@SuppressWarnings("rawtypes")
public static StateFactory opaque(CqlRowMapper mapper) {
    Options<OpaqueValue> options = new Options<OpaqueValue>();
    return opaque(mapper, options);
}
 

public static StormTopology buildTopology(String hbaseRoot){
    Fields fields = new Fields("word", "count");
    FixedBatchSpout spout = new FixedBatchSpout(fields, 4,
            new Values("storm", 1),
            new Values("trident", 1),
            new Values("needs", 1),
            new Values("javadoc", 1)
    );
    spout.setCycle(true);

    TridentHBaseMapper tridentHBaseMapper = new SimpleTridentHBaseMapper()
            .withColumnFamily("cf")
            .withColumnFields(new Fields("word"))
            .withCounterFields(new Fields("count"))
            .withRowKeyField("word");

    HBaseValueMapper rowToStormValueMapper = new WordCountValueMapper();

    HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria();
    projectionCriteria.addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count"));

    HBaseState.Options options = new HBaseState.Options()
            .withConfigKey(hbaseRoot)
            .withDurability(Durability.SYNC_WAL)
            .withMapper(tridentHBaseMapper)
            .withProjectionCriteria(projectionCriteria)
            .withRowToStormValueMapper(rowToStormValueMapper)
            .withTableName("WordCount");

    StateFactory factory = new HBaseStateFactory(options);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    stream.partitionPersist(factory, fields,  new HBaseUpdater(), new Fields());

    TridentState state = topology.newStaticState(factory);
    stream = stream.stateQuery(state, new Fields("word"), new HBaseQuery(), new Fields("columnName","columnValue"));
    stream.each(new Fields("word","columnValue"), new PrintFunction(), new Fields());
    return topology.build();
}