Java Code Examples for backtype.storm.generated.ComponentCommon#get_json_conf()

The following examples show how to use backtype.storm.generated.ComponentCommon#get_json_conf() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NimbusUtils.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
public static Integer componentParalism(Map stormConf, ComponentCommon common) {
    Map mergeMap = new HashMap();
    mergeMap.putAll(stormConf);

    String jsonConfString = common.get_json_conf();
    if (jsonConfString != null) {
        Map componentMap = (Map) JStormUtils.from_json(jsonConfString);
        mergeMap.putAll(componentMap);
    }

    Integer taskNum = common.get_parallelism_hint();

    // don't get taskNum from component configuraiton
    // skip .setTaskNum
    // Integer taskNum = null;
    // Object taskNumObject = mergeMap.get(Config.TOPOLOGY_TASKS);
    // if (taskNumObject != null) {
    // taskNum = JStormUtils.parseInt(taskNumObject);
    // } else {
    // taskNum = common.get_parallelism_hint();
    // if (taskNum == null) {
    // taskNum = Integer.valueOf(1);
    // }
    // }

    Object maxTaskParalismObject = mergeMap.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM);
    if (maxTaskParalismObject == null) {
        return taskNum;
    } else {
        int maxTaskParalism = JStormUtils.parseInt(maxTaskParalismObject);

        return Math.min(maxTaskParalism, taskNum);
    }

}
Example 2
Source File: GeneralTopologyContext.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
public int maxTopologyMessageTimeout() {
    Integer max = Utils.getInt(_stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
    for (String spout : getRawTopology().get_spouts().keySet()) {
        ComponentCommon common = getComponentCommon(spout);
        String jsonConf = common.get_json_conf();
        if (jsonConf != null) {
            Map conf = (Map) Utils.from_json(jsonConf);
            Object comp = conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS);
            if (comp != null) {
                max = Math.max(Utils.getInt(comp), max);
            }
        }
    }
    return max;
}
Example 3
Source File: NimbusUtils.java    From jstorm with Apache License 2.0 4 votes vote down vote up 
/**
 * Normalize stormConf
 *
 * @param conf      cluster conf
 * @param stormConf storm topology conf
 * @param topology  storm topology
 * @return normalized conf
 * @throws Exception
 */
@SuppressWarnings("rawtypes")
public static Map normalizeConf(Map conf, Map stormConf, StormTopology topology) throws Exception {

    List kryoRegisterList = new ArrayList();
    List kryoDecoratorList = new ArrayList();

    Map totalConf = new HashMap();
    totalConf.putAll(conf);
    totalConf.putAll(stormConf);

    Object totalRegister = totalConf.get(Config.TOPOLOGY_KRYO_REGISTER);
    if (totalRegister != null) {
        LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", TOPOLOGY_KRYO_REGISTER" +
                totalRegister.getClass().getName());

        JStormUtils.mergeList(kryoRegisterList, totalRegister);
    }

    Object totalDecorator = totalConf.get(Config.TOPOLOGY_KRYO_DECORATORS);
    if (totalDecorator != null) {
        LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", TOPOLOGY_KRYO_DECORATOR" +
                totalDecorator.getClass().getName());
        JStormUtils.mergeList(kryoDecoratorList, totalDecorator);
    }

    Set<String> cids = ThriftTopologyUtils.getComponentIds(topology);
    for (Iterator it = cids.iterator(); it.hasNext(); ) {
        String componentId = (String) it.next();

        ComponentCommon common = ThriftTopologyUtils.getComponentCommon(topology, componentId);
        String json = common.get_json_conf();
        if (json == null) {
            continue;
        }
        Map mtmp = (Map) JStormUtils.from_json(json);
        if (mtmp == null) {
            StringBuilder sb = new StringBuilder();
            sb.append("Failed to deserialize " + componentId);
            sb.append(" json configuration: ");
            sb.append(json);
            LOG.info(sb.toString());
            throw new Exception(sb.toString());
        }

        Object componentKryoRegister = mtmp.get(Config.TOPOLOGY_KRYO_REGISTER);

        if (componentKryoRegister != null) {
            LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", componentId:" + componentId +
                    ", TOPOLOGY_KRYO_REGISTER" + componentKryoRegister.getClass().getName());

            JStormUtils.mergeList(kryoRegisterList, componentKryoRegister);
        }

        Object componentDecorator = mtmp.get(Config.TOPOLOGY_KRYO_DECORATORS);
        if (componentDecorator != null) {
            LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", componentId:" + componentId +
                    ", TOPOLOGY_KRYO_DECORATOR" + componentDecorator.getClass().getName());
            JStormUtils.mergeList(kryoDecoratorList, componentDecorator);
        }

        boolean isTransactionTopo = JStormUtils.parseBoolean(mtmp.get(ConfigExtension.TRANSACTION_TOPOLOGY), false);
        if (isTransactionTopo)
            stormConf.put(ConfigExtension.TRANSACTION_TOPOLOGY, true);
    }

    Map kryoRegisterMap = mapifySerializations(kryoRegisterList);
    List decoratorList = JStormUtils.distinctList(kryoDecoratorList);

    Integer ackerNum = JStormUtils.parseInt(totalConf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);

    Map rtn = new HashMap();
    //ensure to be cluster_mode
    rtn.put(Config.STORM_CLUSTER_MODE, conf.get(Config.STORM_CLUSTER_MODE));
    rtn.putAll(stormConf);
    rtn.put(Config.TOPOLOGY_KRYO_DECORATORS, decoratorList);
    rtn.put(Config.TOPOLOGY_KRYO_REGISTER, kryoRegisterMap);
    rtn.put(Config.TOPOLOGY_ACKER_EXECUTORS, ackerNum);
    rtn.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, totalConf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM));

    return rtn;
}