Java Code Examples for backtype.storm.utils.Utils#DEFAULT_STREAM_ID

The following examples show how to use backtype.storm.utils.Utils#DEFAULT_STREAM_ID . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ConfigurableIngestTopologyTest.java    From cognition with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureStreamGrouping_StreamFieldsGrouping(
    @Injectable String prevComponent,
    @Injectable Configuration boltConf,
    @Injectable BoltDeclarer declarer) throws Exception {
  new Expectations(topology) {{
    boltConf.getString(STREAM_GROUPING_CONF_TYPE, STREAM_GROUPING_LOCAL_OR_SHUFFLE);
    result = STREAM_GROUPING_FIELDS;
    boltConf.getString(STREAM_ID, Utils.DEFAULT_STREAM_ID);
    result = Utils.DEFAULT_STREAM_ID;
    topology.configureStreamFieldsGrouping(prevComponent, Utils.DEFAULT_STREAM_ID, boltConf, declarer);
  }};

  topology.configureStreamGrouping(prevComponent, boltConf, declarer);
}
Example 2
Source File: ConfigurableIngestTopologyTest.java    From cognition with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureStreamGrouping_localOrShuffleGrouping(
    @Injectable String prevComponent,
    @Injectable Configuration boltConf,
    @Injectable BoltDeclarer declarer) throws Exception {
  new Expectations(topology) {{
    boltConf.getString(STREAM_GROUPING_CONF_TYPE, STREAM_GROUPING_LOCAL_OR_SHUFFLE);
    result = STREAM_GROUPING_LOCAL_OR_SHUFFLE;
    boltConf.getString(STREAM_ID, Utils.DEFAULT_STREAM_ID);
    result = Utils.DEFAULT_STREAM_ID;
    declarer.localOrShuffleGrouping(prevComponent, Utils.DEFAULT_STREAM_ID);
  }};

  topology.configureStreamGrouping(prevComponent, boltConf, declarer);
}
Example 3
Source File: ConfigurableIngestTopologyTest.java    From cognition with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigureStreamGrouping_default(
    @Injectable String prevComponent,
    @Injectable Configuration boltConf,
    @Injectable BoltDeclarer declarer) throws Exception {
  new Expectations(topology) {{
    boltConf.getString(STREAM_GROUPING_CONF_TYPE, STREAM_GROUPING_LOCAL_OR_SHUFFLE);
    result = STREAM_GROUPING_SHUFFLE;
    boltConf.getString(STREAM_ID, Utils.DEFAULT_STREAM_ID);
    result = Utils.DEFAULT_STREAM_ID;
    declarer.shuffleGrouping(prevComponent, Utils.DEFAULT_STREAM_ID);
  }};

  topology.configureStreamGrouping(prevComponent, boltConf, declarer);
}
Example 4
Source File: FluxBuilder.java    From flux with Apache License 2.0 4 votes vote down vote up 
/**
 * @param context
 * @param builder
 */
private static void buildStreamDefinitions(ExecutionContext context, TopologyBuilder builder)
        throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException,
        IllegalAccessException {
    TopologyDef topologyDef = context.getTopologyDef();
    // process stream definitions
    for (StreamDef stream : topologyDef.getStreams()) {
        Object boltObj = context.getBolt(stream.getTo());
        BoltDeclarer declarer = null;
        if (boltObj instanceof IRichBolt) {
            declarer = builder.setBolt(stream.getTo(),
                    (IRichBolt) boltObj,
                    topologyDef.parallelismForBolt(stream.getTo()));
        } else if (boltObj instanceof IBasicBolt) {
            declarer = builder.setBolt(
                    stream.getTo(),
                    (IBasicBolt) boltObj,
                    topologyDef.parallelismForBolt(stream.getTo()));
        } else {
            throw new IllegalArgumentException("Class does not appear to be a bolt: " +
                    boltObj.getClass().getName());
        }

        GroupingDef grouping = stream.getGrouping();
        // if the streamId is defined, use it for the grouping, otherwise assume storm's default stream
        String streamId = (grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId());


        switch (grouping.getType()) {
            case SHUFFLE:
                declarer.shuffleGrouping(stream.getFrom(), streamId);
                break;
            case FIELDS:
                //TODO check for null grouping args
                declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(grouping.getArgs()));
                break;
            case ALL:
                declarer.allGrouping(stream.getFrom(), streamId);
                break;
            case DIRECT:
                declarer.directGrouping(stream.getFrom(), streamId);
                break;
            case GLOBAL:
                declarer.globalGrouping(stream.getFrom(), streamId);
                break;
            case LOCAL_OR_SHUFFLE:
                declarer.localOrShuffleGrouping(stream.getFrom(), streamId);
                break;
            case NONE:
                declarer.noneGrouping(stream.getFrom(), streamId);
                break;
            case CUSTOM:
                declarer.customGrouping(stream.getFrom(), streamId,
                        buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), context));
                break;
            default:
                throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
        }
    }
}
Example 5
Source File: FluxBuilder.java    From jstorm with Apache License 2.0 4 votes vote down vote up 
/**
 * @param context
 * @param builder
 */
private static void buildStreamDefinitions(ExecutionContext context, TopologyBuilder builder)
        throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException,
        IllegalAccessException {
    TopologyDef topologyDef = context.getTopologyDef();
    // process stream definitions
    HashMap<String, BoltDeclarer> declarers = new HashMap<String, BoltDeclarer>();
    for (StreamDef stream : topologyDef.getStreams()) {
        Object boltObj = context.getBolt(stream.getTo());
        BoltDeclarer declarer = declarers.get(stream.getTo());
        if (boltObj instanceof IRichBolt) {
            if(declarer == null) {
                declarer = builder.setBolt(stream.getTo(),
                        (IRichBolt) boltObj,
                        topologyDef.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IBasicBolt) {
            if(declarer == null) {
                declarer = builder.setBolt(
                        stream.getTo(),
                        (IBasicBolt) boltObj,
                        topologyDef.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        }  else {
            throw new IllegalArgumentException("Class does not appear to be a bolt: " +
                    boltObj.getClass().getName());
        }
        //sorry, jstorm don't support IWindowedBolt
        /*else if (boltObj instanceof IWindowedBolt) {
            if(declarer == null) {
                declarer = builder.setBolt(
                        stream.getTo(),
                        (IWindowedBolt) boltObj,
                        topologyDef.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        }*/

        GroupingDef grouping = stream.getGrouping();
        // if the streamId is defined, use it for the grouping, otherwise assume storm's default stream
        String streamId = (grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId());


        switch (grouping.getType()) {
            case SHUFFLE:
                declarer.shuffleGrouping(stream.getFrom(), streamId);
                break;
            case FIELDS:
                //TODO check for null grouping args
                declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(grouping.getArgs()));
                break;
            case ALL:
                declarer.allGrouping(stream.getFrom(), streamId);
                break;
            case DIRECT:
                declarer.directGrouping(stream.getFrom(), streamId);
                break;
            case GLOBAL:
                declarer.globalGrouping(stream.getFrom(), streamId);
                break;
            case LOCAL_OR_SHUFFLE:
                declarer.localOrShuffleGrouping(stream.getFrom(), streamId);
                break;
            case NONE:
                declarer.noneGrouping(stream.getFrom(), streamId);
                break;
            case CUSTOM:
                declarer.customGrouping(stream.getFrom(), streamId,
                        buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), context));
                break;
            case LOCAL_FIRST:
                declarer.localFirstGrouping(stream.getFrom(), streamId);
                break;
            case PARTIAL_KEY:
                declarer.partialKeyGrouping(stream.getFrom(), streamId, new Fields(grouping.getArgs()));
                break;
            default:
                throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
        }
    }
}
Example 6
Source File: JsonSerializer.java    From jstorm with Apache License 2.0 4 votes vote down vote up 
public ShellMsg readShellMsg() throws IOException, NoOutputException {
    JSONObject msg = (JSONObject) readMessage();
    ShellMsg shellMsg = new ShellMsg();

    String command = (String) msg.get("command");
    shellMsg.setCommand(command);

    Object id = msg.get("id");
    shellMsg.setId(id);

    String log = (String) msg.get("msg");
    shellMsg.setMsg(log);

    String stream = (String) msg.get("stream");
    if (stream == null)
        stream = Utils.DEFAULT_STREAM_ID;
    shellMsg.setStream(stream);

    Object taskObj = msg.get("task");
    if (taskObj != null) {
        shellMsg.setTask((Long) taskObj);
    } else {
        shellMsg.setTask(0);
    }

    Object need_task_ids = msg.get("need_task_ids");
    if (need_task_ids == null || (Boolean) need_task_ids) {
        shellMsg.setNeedTaskIds(true);
    } else {
        shellMsg.setNeedTaskIds(false);
    }

    shellMsg.setTuple((List) msg.get("tuple"));

    //List<Tuple> anchors = new ArrayList<Tuple>();
    Object anchorObj = msg.get("anchors");
    if (anchorObj != null) {
        if (anchorObj instanceof String) {
            anchorObj = Arrays.asList(anchorObj);
        }
        for (Object o : (List) anchorObj) {
            shellMsg.addAnchor((String) o);
        }
    }

    Object nameObj = msg.get("name");
    String metricName = null;
    if (nameObj != null && nameObj instanceof String) {
        metricName = (String) nameObj;
    }
    shellMsg.setMetricName(metricName);

    Object paramsObj = msg.get("params");
    shellMsg.setMetricParams(paramsObj);

    if (command.equals("log")) {
        Object logLevelObj = msg.get("level");
        if (logLevelObj != null && logLevelObj instanceof Long) {
            long logLevel = (Long) logLevelObj;
            shellMsg.setLogLevel((int) logLevel);
        }
    }

    return shellMsg;
}
Example 7
Source File: FixedTuple.java    From jstorm with Apache License 2.0 4 votes vote down vote up 
public FixedTuple(List<Object> values) {
    this.stream = Utils.DEFAULT_STREAM_ID;
    this.values = values;
}