backtype.storm.generated.ComponentCommon Java Examples

protected void initCommon(String id, IComponent component, Number parallelism) throws IllegalArgumentException {
    ComponentCommon common = new ComponentCommon();
    common.set_inputs(new HashMap<GlobalStreamId, Grouping>());
    if (parallelism != null) {
        int dop = parallelism.intValue();
        if(dop < 1) {
            throw new IllegalArgumentException("Parallelism must be positive.");
        }
        common.set_parallelism_hint(dop);
    } else {
        common.set_parallelism_hint(1);
    }
    Map conf = component.getComponentConfiguration();
    if(conf!=null) common.set_json_conf(JSONValue.toJSONString(conf));
    _commons.put(id, common);
}
 

public static Integer componentParalism(Map stormConf, ComponentCommon common) {
    Map mergeMap = new HashMap();
    mergeMap.putAll(stormConf);

    String jsonConfString = common.get_json_conf();
    if (jsonConfString != null) {
        Map componentMap = (Map) JStormUtils.from_json(jsonConfString);
        mergeMap.putAll(componentMap);
    }

    Integer taskNum = common.get_parallelism_hint();

    // don't get taskNum from component configuraiton
    // skip .setTaskNum
    // Integer taskNum = null;
    // Object taskNumObject = mergeMap.get(Config.TOPOLOGY_TASKS);
    // if (taskNumObject != null) {
    // taskNum = JStormUtils.parseInt(taskNumObject);
    // } else {
    // taskNum = common.get_parallelism_hint();
    // if (taskNum == null) {
    // taskNum = Integer.valueOf(1);
    // }
    // }

    Object maxTaskParalismObject = mergeMap.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM);
    if (maxTaskParalismObject == null) {
        return taskNum;
    } else {
        int maxTaskParalism = JStormUtils.parseInt(maxTaskParalismObject);

        return Math.min(maxTaskParalism, taskNum);
    }

}
 

public static Set<String> getUpstreamComponents(String componentId, TopologyContext context) {
    Set<String> upstreamComponents = new HashSet<>();
    ComponentCommon componentCommon = Utils.getComponentCommon(context.getRawTopology(), componentId);
    Set<GlobalStreamId> input = componentCommon.get_inputs().keySet();
    for (GlobalStreamId stream : input) {
        upstreamComponents.add(stream.get_componentId());
    }
    return upstreamComponents;
}
 

private static boolean hasDownstreamComponent(StormTopology topology, String outComponentId, String outStreamId) {
    boolean ret = false;
    for (String componentId : ThriftTopologyUtils.getComponentIds(topology)) {
        ComponentCommon componentCommon = Utils.getComponentCommon(topology, componentId);
        Set<GlobalStreamId> inputs = componentCommon.get_inputs().keySet();
        for (GlobalStreamId input : inputs) {
            if (input.get_componentId().equals(outComponentId) && input.get_streamId().equals(outStreamId))
                return true;
        }
    }
    return ret;
}
 

public IdDictionary(StormTopology topology) {
    List<String> componentNames = new ArrayList<>(topology.get_spouts().keySet());
    componentNames.addAll(topology.get_bolts().keySet());
    componentNames.addAll(topology.get_state_spouts().keySet());

    for (String name : componentNames) {
        ComponentCommon common = Utils.getComponentCommon(topology, name);
        List<String> streams = new ArrayList<>(common.get_streams().keySet());
        streamNametoId.put(name, idify(streams));
        streamIdToName.put(name, Utils.reverseMap(streamNametoId.get(name)));
    }
}
 

public static ComponentCommon getComponentCommon(StormTopology topology, String id) {
    if (topology.get_spouts().containsKey(id)) {
        return topology.get_spouts().get(id).get_common();
    }
    if (topology.get_bolts().containsKey(id)) {
        return topology.get_bolts().get(id).get_common();
    }
    if (topology.get_state_spouts().containsKey(id)) {
        return topology.get_state_spouts().get(id).get_common();
    }
    throw new IllegalArgumentException("Could not find component with id " + id);
}
 

public int maxTopologyMessageTimeout() {
    Integer max = Utils.getInt(_stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
    for (String spout : getRawTopology().get_spouts().keySet()) {
        ComponentCommon common = getComponentCommon(spout);
        String jsonConf = common.get_json_conf();
        if (jsonConf != null) {
            Map conf = (Map) Utils.from_json(jsonConf);
            Object comp = conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS);
            if (comp != null) {
                max = Math.max(Utils.getInt(comp), max);
            }
        }
    }
    return max;
}
 

/**
 * Add watermark stream to source components of window bolts
 */
private void maybeAddWatermarkInputs(ComponentCommon common, IRichBolt bolt) {
    if (bolt instanceof WindowedBoltExecutor) {
        Set<String> comps = new HashSet<>();
        for (GlobalStreamId globalStreamId : common.get_inputs().keySet()) {
            comps.add(globalStreamId.get_componentId());
        }

        for (String comp : comps) {
            common.put_to_inputs(
                    new GlobalStreamId(comp, Common.WATERMARK_STREAM_ID),
                    Grouping.all(new NullStruct()));
        }
    }
}
 

/**
 * For bolts that has incoming streams from spouts (the root bolts),
 * add checkpoint stream from checkpoint spout to its input. For other bolts,
 * add checkpoint stream from the previous bolt to its input.
 */
private void addCheckPointInputs(ComponentCommon component) {
    Set<GlobalStreamId> checkPointInputs = new HashSet<>();
    for (GlobalStreamId inputStream : component.get_inputs().keySet()) {
        String sourceId = inputStream.get_componentId();
        if (_spouts.containsKey(sourceId)) {
            checkPointInputs.add(new GlobalStreamId(CheckpointSpout.CHECKPOINT_COMPONENT_ID, CheckpointSpout.CHECKPOINT_STREAM_ID));
        } else {
            checkPointInputs.add(new GlobalStreamId(sourceId, CheckpointSpout.CHECKPOINT_STREAM_ID));
        }
    }
    for (GlobalStreamId streamId : checkPointInputs) {
        component.put_to_inputs(streamId, Grouping.all(new NullStruct()));
    }
}
 

private ComponentCommon getComponentCommon(String id, IComponent component) {
    ComponentCommon ret = new ComponentCommon(_commons.get(id));

    OutputFieldsGetter getter = new OutputFieldsGetter();
    component.declareOutputFields(getter);
    // declare watermark stream for all components
    getter.declareStream(Common.WATERMARK_STREAM_ID, new Fields("watermark"));
    ret.set_streams(getter.getFieldsDeclaration());
    return ret;
}
 

public static void verifyParallelism(ComponentCommon component, int expected) {
  assertThat(component.get_parallelism_hint()).isEqualTo(expected);
}
 

/**
 * Normalize stormConf
 *
 * @param conf      cluster conf
 * @param stormConf storm topology conf
 * @param topology  storm topology
 * @return normalized conf
 * @throws Exception
 */
@SuppressWarnings("rawtypes")
public static Map normalizeConf(Map conf, Map stormConf, StormTopology topology) throws Exception {

    List kryoRegisterList = new ArrayList();
    List kryoDecoratorList = new ArrayList();

    Map totalConf = new HashMap();
    totalConf.putAll(conf);
    totalConf.putAll(stormConf);

    Object totalRegister = totalConf.get(Config.TOPOLOGY_KRYO_REGISTER);
    if (totalRegister != null) {
        LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", TOPOLOGY_KRYO_REGISTER" +
                totalRegister.getClass().getName());

        JStormUtils.mergeList(kryoRegisterList, totalRegister);
    }

    Object totalDecorator = totalConf.get(Config.TOPOLOGY_KRYO_DECORATORS);
    if (totalDecorator != null) {
        LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", TOPOLOGY_KRYO_DECORATOR" +
                totalDecorator.getClass().getName());
        JStormUtils.mergeList(kryoDecoratorList, totalDecorator);
    }

    Set<String> cids = ThriftTopologyUtils.getComponentIds(topology);
    for (Iterator it = cids.iterator(); it.hasNext(); ) {
        String componentId = (String) it.next();

        ComponentCommon common = ThriftTopologyUtils.getComponentCommon(topology, componentId);
        String json = common.get_json_conf();
        if (json == null) {
            continue;
        }
        Map mtmp = (Map) JStormUtils.from_json(json);
        if (mtmp == null) {
            StringBuilder sb = new StringBuilder();
            sb.append("Failed to deserialize " + componentId);
            sb.append(" json configuration: ");
            sb.append(json);
            LOG.info(sb.toString());
            throw new Exception(sb.toString());
        }

        Object componentKryoRegister = mtmp.get(Config.TOPOLOGY_KRYO_REGISTER);

        if (componentKryoRegister != null) {
            LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", componentId:" + componentId +
                    ", TOPOLOGY_KRYO_REGISTER" + componentKryoRegister.getClass().getName());

            JStormUtils.mergeList(kryoRegisterList, componentKryoRegister);
        }

        Object componentDecorator = mtmp.get(Config.TOPOLOGY_KRYO_DECORATORS);
        if (componentDecorator != null) {
            LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", componentId:" + componentId +
                    ", TOPOLOGY_KRYO_DECORATOR" + componentDecorator.getClass().getName());
            JStormUtils.mergeList(kryoDecoratorList, componentDecorator);
        }

        boolean isTransactionTopo = JStormUtils.parseBoolean(mtmp.get(ConfigExtension.TRANSACTION_TOPOLOGY), false);
        if (isTransactionTopo)
            stormConf.put(ConfigExtension.TRANSACTION_TOPOLOGY, true);
    }

    Map kryoRegisterMap = mapifySerializations(kryoRegisterList);
    List decoratorList = JStormUtils.distinctList(kryoDecoratorList);

    Integer ackerNum = JStormUtils.parseInt(totalConf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);

    Map rtn = new HashMap();
    //ensure to be cluster_mode
    rtn.put(Config.STORM_CLUSTER_MODE, conf.get(Config.STORM_CLUSTER_MODE));
    rtn.putAll(stormConf);
    rtn.put(Config.TOPOLOGY_KRYO_DECORATORS, decoratorList);
    rtn.put(Config.TOPOLOGY_KRYO_REGISTER, kryoRegisterMap);
    rtn.put(Config.TOPOLOGY_ACKER_EXECUTORS, ackerNum);
    rtn.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, totalConf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM));

    return rtn;
}
 

public ComponentCommon getComponentCommon(String componentId) {
    return ThriftTopologyUtils.getComponentCommon(getRawTopology(), componentId);
}
 

private void maybeAddCheckpointInputs(ComponentCommon common) {
    if (hasStatefulBolt) {
        addCheckPointInputs(common);
    }
}