Java Code Examples for storm.trident.TridentTopology#newStream()

public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();

    GameState exampleRecursiveState = GameState.playAtRandom(new Board(), "X");
    LOG.info("SIMULATED LEAF NODE : [" + exampleRecursiveState.getBoard() + "] w/ state [" + exampleRecursiveState + "]");

    // Scoring Queue / Spout
    LocalQueueEmitter<GameState> scoringSpoutEmitter = new LocalQueueEmitter<GameState>("ScoringQueue");
    scoringSpoutEmitter.enqueue(exampleRecursiveState);
    LocalQueueSpout<GameState> scoringSpout = new LocalQueueSpout<GameState>(scoringSpoutEmitter);

    Stream inputStream = topology.newStream("scoring", scoringSpout);

    inputStream.each(new Fields("gamestate"), new isEndGame())
            .each(new Fields("gamestate"),
                    new ScoreFunction(),
                    new Fields("board", "score", "player"))
            .each(new Fields("board", "score", "player"), new ScoreUpdater(), new Fields());
    return topology.build();
}
 

public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    StateFactory clickThruMemory = new MemoryMapState.Factory();
    ClickThruSpout spout = new ClickThruSpout();
    Stream inputStream = topology.newStream("clithru", spout);
    TridentState clickThruState = inputStream.each(new Fields("username", "campaign", "product", "click"), new Filter("click", "true"))
            .each(new Fields("username", "campaign", "product", "click"), new Distinct())
            .groupBy(new Fields("campaign"))
            .persistentAggregate(clickThruMemory, new Count(), new Fields("click_thru_count"));

    inputStream.groupBy(new Fields("campaign"))
            .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("impression_count"))
            .newValuesStream()
            .stateQuery(clickThruState, new Fields("campaign"), new MapGet(), new Fields("click_thru_count"))
            .each(new Fields("campaign", "impression_count", "click_thru_count"), new CampaignEffectiveness(), new Fields(""));

    return topology.build();
}
 

public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();

    // Work Queue / Spout
    LocalQueueEmitter<GameState> workSpoutEmitter = new LocalQueueEmitter<GameState>("WorkQueue");
    LocalQueueSpout<GameState> workSpout = new LocalQueueSpout<GameState>(workSpoutEmitter);
    GameState initialState = new GameState(new Board(), new ArrayList<Board>(), "X");
    workSpoutEmitter.enqueue(initialState);

    // Scoring Queue / Spout
    LocalQueueEmitter<GameState> scoringSpoutEmitter = new LocalQueueEmitter<GameState>("ScoringQueue");

    Stream inputStream = topology.newStream("gamestate", workSpout);

    inputStream.each(new Fields("gamestate"), new isEndGame())
            .each(new Fields("gamestate"),
                    new LocalQueuerFunction<GameState>(scoringSpoutEmitter),
                    new Fields(""));

    inputStream.each(new Fields("gamestate"), new GenerateBoards(), new Fields("children"))
            .each(new Fields("children"),
                    new LocalQueuerFunction<GameState>(workSpoutEmitter),
                    new Fields());

    return topology.build();
}
 

public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    TwitterSpout spout = new TwitterSpout();
    Stream inputStream = topology.newStream("nlp", spout);
    try {
        inputStream.each(new Fields("tweet"), new TweetSplitterFunction(), new Fields("word"))
                .each(new Fields("searchphrase", "tweet", "word"), new WordFrequencyFunction(), new Fields("baseline"))
                .each(new Fields("searchphrase", "tweet", "word", "baseline"), new PersistenceFunction(), new Fields("none"))
                .partitionPersist(new DruidStateFactory(), new Fields("searchphrase", "tweet", "word", "baseline"), new DruidStateUpdater());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return topology.build();
}
 

public static StormTopology buildTopology() {
    TridentTopology topology = new TridentTopology();
    DiagnosisEventSpout spout = new DiagnosisEventSpout();
    Stream inputStream = topology.newStream("event", spout);

    inputStream.each(new Fields("event"), new DiseaseFilter())
            .each(new Fields("event"), new CityAssignment(), new Fields("city"))
            .each(new Fields("event", "city"), new HourAssignment(), new Fields("hour", "cityDiseaseHour"))
            .groupBy(new Fields("cityDiseaseHour"))
            .persistentAggregate(new OutbreakTrendFactory(), new Count(), new Fields("count")).newValuesStream()
            .each(new Fields("cityDiseaseHour", "count"), new OutbreakDetector(), new Fields("alert"))
            .each(new Fields("alert"), new DispatchAlert(), new Fields());
    return topology.build();
}
 

public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    FixEventSpout spout = new FixEventSpout();
    Stream inputStream = topology.newStream("message", spout);

    inputStream.each(new Fields("message"), new MessageTypeFilter())
            .partitionPersist(new DruidStateFactory(), new Fields("message"), new DruidStateUpdater());
    return topology.build();
}
 

public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    SalesSpout spout = new SalesSpout();
    Stream inputStream = topology.newStream("sales", spout);
    SalesMapper mapper = new SalesMapper();
    inputStream.partitionPersist(
            new CassandraCqlIncrementalStateFactory<String, Number>(new Sum(), mapper),
            new Fields("price", "state", "product"),
            new CassandraCqlIncrementalStateUpdater<String, Number>());
    return topology.build();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    SimpleUpdateSpout spout = new SimpleUpdateSpout();
    Stream inputStream = topology.newStream("test", spout);
    SimpleUpdateMapper mapper = new SimpleUpdateMapper();
    inputStream.partitionPersist(new CassandraCqlStateFactory(ConsistencyLevel.ONE), new Fields("test"), new CassandraCqlStateUpdater(mapper));
    // inputStream.each(new Fields("test"), new Debug());
    return topology.build();
}
 

public static StormTopology buildTopology(String hdfsUrl){
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields hdfsFields = new Fields("sentence", "key");

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/trident")
            .withPrefix("trident")
            .withExtension(".txt");

    RecordFormat recordFormat = new DelimitedRecordFormat()
            .withFields(hdfsFields);

    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    HdfsState.Options options = new HdfsState.HdfsFileOptions()
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(recordFormat)
            .withRotationPolicy(rotationPolicy)
            .withFsUrl(hdfsUrl);

    StateFactory factory = new HdfsStateFactory().withOptions(options);

    TridentState state = stream
            .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

    return topology.build();
}
 

public static StormTopology buildTopology(String hdfsUrl){
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields hdfsFields = new Fields("sentence", "key");

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/trident")
            .withPrefix("trident")
            .withExtension(".seq");

    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions()
            .withFileNameFormat(fileNameFormat)
            .withSequenceFormat(new DefaultSequenceFormat("key", "sentence"))
            .withRotationPolicy(rotationPolicy)
            .withFsUrl(hdfsUrl)
            .addRotationAction(new MoveFileAction().toDestination("/dest2/"));

    StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);

    TridentState state = stream
            .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

    return topology.build();
}
 

public static StormTopology buildTopology(String hbaseRoot){
    Fields fields = new Fields("word", "count");
    FixedBatchSpout spout = new FixedBatchSpout(fields, 4,
            new Values("storm", 1),
            new Values("trident", 1),
            new Values("needs", 1),
            new Values("javadoc", 1)
    );
    spout.setCycle(true);

    TridentHBaseMapper tridentHBaseMapper = new SimpleTridentHBaseMapper()
            .withColumnFamily("cf")
            .withColumnFields(new Fields("word"))
            .withCounterFields(new Fields("count"))
            .withRowKeyField("word");

    HBaseValueMapper rowToStormValueMapper = new WordCountValueMapper();

    HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria();
    projectionCriteria.addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count"));

    HBaseState.Options options = new HBaseState.Options()
            .withConfigKey(hbaseRoot)
            .withDurability(Durability.SYNC_WAL)
            .withMapper(tridentHBaseMapper)
            .withProjectionCriteria(projectionCriteria)
            .withRowToStormValueMapper(rowToStormValueMapper)
            .withTableName("WordCount");

    StateFactory factory = new HBaseStateFactory(options);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    stream.partitionPersist(factory, fields,  new HBaseUpdater(), new Fields());

    TridentState state = topology.newStaticState(factory);
    stream = stream.stateQuery(state, new Fields("word"), new HBaseQuery(), new Fields("columnName","columnValue"));
    stream.each(new Fields("word","columnValue"), new PrintFunction(), new Fields());
    return topology.build();
}