backtype.storm.Constants Java Examples

public static StormTopology add_system_components(StormTopology topology) {
    // generate inputs
    Map<GlobalStreamId, Grouping> inputs = new HashMap<>();

    // generate outputs
    HashMap<String, StreamInfo> outputs = new HashMap<>();
    //ArrayList<String> fields = new ArrayList<String>();

    outputs.put(Constants.SYSTEM_TICK_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("rate_secs")));
    outputs.put(Constants.METRICS_TICK_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("interval")));
    outputs.put(Constants.CREDENTIALS_CHANGED_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("creds")));

    // ComponentCommon common = new ComponentCommon(inputs, outputs);

    IBolt ackerbolt = new SystemBolt();
    Bolt bolt = Thrift.mkBolt(inputs, ackerbolt, outputs, 0);
    topology.put_to_bolts(Constants.SYSTEM_COMPONENT_ID, bolt);

    add_system_streams(topology);

    return topology;

}
 

public void prepare(Map config, TopologyContext context, OutputCollector collector) {
    TimeCacheMap.ExpiredCallback<Object, TrackingInfo> callback = null;
    if (_delegate instanceof TimeoutCallback) {
        callback = new TimeoutItems();
    }
    _tracked = new TimeCacheMap<>(context.maxTopologyMessageTimeout(), callback);
    _collector = collector;
    _delegate.prepare(config, context, new OutputCollector(new CoordinatedOutputCollector(collector)));
    for (String component : Utils.get(context.getThisTargets(), Constants.COORDINATED_STREAM_ID, new HashMap<String, Grouping>()).keySet()) {
        for (Integer task : context.getComponentTasks(component)) {
            _countOutTasks.add(task);
        }
    }
    if (!_sourceArgs.isEmpty()) {
        _numSourceReports = 0;
        for (Entry<String, SourceArgs> entry : _sourceArgs.entrySet()) {
            if (entry.getValue().singleCount) {
                _numSourceReports += 1;
            } else {
                _numSourceReports += context.getComponentTasks(entry.getKey()).size();
            }
        }
    }
}
 

private BoltMsg createHeartbeatBoltMessage(String genId) {
    BoltMsg msg = new BoltMsg();
    msg.setId(genId);
    msg.setTask(Constants.SYSTEM_TASK_ID);
    msg.setStream(HEARTBEAT_STREAM_ID);
    msg.setTuple(new ArrayList<>());
    return msg;
}
 

/**
 * Gets the component id for the specified task id.
 * The component id maps to a component id specified for a Spout or Bolt in the topology definition.
 *
 * @param taskId the task id
 * @return the component id for the input task id
 */
public String getComponentId(int taskId) {
    if (taskId == Constants.SYSTEM_TASK_ID) {
        return Constants.SYSTEM_COMPONENT_ID;
    } else {
        return _taskToComponent.get(taskId);
    }
}
 

private TupleType getTupleType(Tuple tuple) {
    if (_idStreamSpec != null && tuple.getSourceGlobalStreamid().equals(_idStreamSpec._id)) {
        return TupleType.ID;
    } else if (!_sourceArgs.isEmpty() && tuple.getSourceStreamId().equals(Constants.COORDINATED_STREAM_ID)) {
        return TupleType.COORD;
    } else {
        return TupleType.REGULAR;
    }
}
 

public LogHandler(Map conf) {
    this.pageSize = ConfigExtension.getLogPageSize(conf);
    logDir = JStormUtils.getLogDir();
    String logDirPath = PathUtils.getCanonicalPath(logDir);
    if (logDirPath == null) {
        accessDirs.add(logDir);
    } else {
        accessDirs.add(logDirPath);
    }

    stormHome = System.getProperty("jstorm.home");
    if (stormHome != null) {
        String stormHomePath = PathUtils.getCanonicalPath(stormHome);
        if (stormHomePath == null) {
            accessDirs.add(stormHome);
        } else {
            accessDirs.add(stormHomePath);
        }
    }

    String confDir = System.getProperty(Constants.JSTORM_CONF_DIR);
    if (!StringUtils.isBlank(confDir)) {
        String confDirPath = PathUtils.getCanonicalPath(confDir);
        if (confDirPath != null) {
            accessDirs.add(confDirPath);
        }
    }

    this.conf = conf;
    LOG.info("logview log dir=" + logDir);
}
 

public BoltExecutors(Task task) {
    super(task);

    this.bolt = (IBolt) task.getTaskObj();

    // create TimeCacheMap
    this.tupleStartTimes = new RotatingMap<>(Acker.TIMEOUT_BUCKET_NUM);
    this.ackerNum = JStormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));

    // create BoltCollector
    BoltCollector outputCollector;
    if (ConfigExtension.isTaskBatchTuple(storm_conf)) {
        outputCollector = new BoltBatchCollector(task, tupleStartTimes, messageTimeoutSecs);
    } else {
        outputCollector = new BoltCollector(task, tupleStartTimes, messageTimeoutSecs);
    }
    this.outputCollector = new OutputCollector(outputCollector);

    //this task don't continue until it bulid connection with topologyMaster
    Integer topologyId = sysTopologyCtx.getTopologyMasterId();
    List<Integer> localWorkerTasks = sysTopologyCtx.getThisWorkerTasks();
    if (topologyId != 0 && !localWorkerTasks.contains(topologyId)) {
        while (getConnection(topologyId) == null) {
            JStormUtils.sleepMs(10);
            LOG.info("this task still is building connection with topology Master");
        }
    }

    taskHbTrigger.setBoltOutputCollector(outputCollector);
    taskHbTrigger.register();
    Object tickFrequency = storm_conf.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_MS);
    if (tickFrequency == null) {
        tickFrequency = storm_conf.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
        if (tickFrequency != null)
            tickFrequency = JStormUtils.parseInt(tickFrequency) * 1000;
    }

    isSystemBolt = Common.isSystemComponent(componentId);
    if (tickFrequency != null && !isSystemBolt) {
        Integer frequency = JStormUtils.parseInt(tickFrequency);
        tickTupleTrigger = new TickTupleTrigger(
                sysTopologyCtx, frequency, idStr + Constants.SYSTEM_TICK_STREAM_ID, controlQueue);
        tickTupleTrigger.register();
    }

    LOG.info("Successfully create BoltExecutors " + idStr);
}
 

public static boolean isTickTuple(Tuple tuple) {
    return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) &&
            tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID);
}
 

public static boolean isTick(Tuple tuple) {
    return tuple != null && Constants.SYSTEM_COMPONENT_ID.equals(tuple.getSourceComponent())
            && Constants.SYSTEM_TICK_STREAM_ID.equals(tuple.getSourceStreamId());
}
 

public void declareOutputFields(OutputFieldsDeclarer declarer) {
    _delegate.declareOutputFields(declarer);
    declarer.declareStream(Constants.COORDINATED_STREAM_ID, true, new Fields("id", "count"));
}
 

private StormTopology createTopology(DRPCSpout spout) {
    final String SPOUT_ID = "spout";
    final String PREPARE_ID = "prepare-request";

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout);
    builder.setBolt(PREPARE_ID, new PrepareRequest()).noneGrouping(SPOUT_ID);
    int i = 0;
    for (; i < _components.size(); i++) {
        Component component = _components.get(i);

        Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
        if (i == 1) {
            source.put(boltId(i - 1), SourceArgs.single());
        } else if (i >= 2) {
            source.put(boltId(i - 1), SourceArgs.all());
        }
        IdStreamSpec idSpec = null;
        if (i == _components.size() - 1 && component.bolt instanceof FinishedCallback) {
            idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
        }
        BoltDeclarer declarer = builder.setBolt(boltId(i), new CoordinatedBolt(component.bolt, source, idSpec), component.parallelism);

        for (Map conf : component.componentConfs) {
            declarer.addConfigurations(conf);
        }

        if (idSpec != null) {
            declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
        }
        if (i == 0 && component.declarations.isEmpty()) {
            declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
        } else {
            String prevId;
            if (i == 0) {
                prevId = PREPARE_ID;
            } else {
                prevId = boltId(i - 1);
            }
            for (InputDeclaration declaration : component.declarations) {
                declaration.declare(prevId, declarer);
            }
        }
        if (i > 0) {
            declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID);
        }
    }

    IRichBolt lastBolt = _components.get(_components.size() - 1).bolt;
    OutputFieldsGetter getter = new OutputFieldsGetter();
    lastBolt.declareOutputFields(getter);
    Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
    if (streams.size() != 1) {
        throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
    }
    String outputStream = streams.keySet().iterator().next();
    List<String> fields = streams.get(outputStream).get_output_fields();
    if (fields.size() != 2) {
        throw new RuntimeException(
                "Output stream of last component in LinearDRPCTopology must contain exactly two fields. The first should be the request id, and the second should be the result.");
    }

    builder.setBolt("JoinResult", new JoinResult(PREPARE_ID)).fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0)))
            .fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
    i++;
    builder.setBolt("ReturnResults", new ReturnResults()).noneGrouping("JoinResult");
    return builder.createTopology();
}
 

public static final boolean isTickTuple(final Tuple tuple) {
  return tuple != null &&
    Constants.SYSTEM_COMPONENT_ID.equals(tuple.getSourceComponent()) &&
    Constants.SYSTEM_TICK_STREAM_ID.equals(tuple.getSourceStreamId());
}
 

@Override
public void updateObject() {
    this.object = new TupleImplExt(topologyContext,
            new Values(TimeUtils.current_time_secs()), (int) Constants.SYSTEM_TASK_ID, Constants.SYSTEM_TICK_STREAM_ID);
}
 

public static boolean isTickTuple(Tuple tuple) {
  return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) && tuple.getSourceStreamId().equals(
      Constants.SYSTEM_TICK_STREAM_ID);
}
 

public static boolean isTickTuple(Tuple tuple) {
  return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID)
      && tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID);
}
 

public static Tuple mockTickTuple() {
  return mockTuple(Constants.SYSTEM_COMPONENT_ID, Constants.SYSTEM_TICK_STREAM_ID);
}
 

public static boolean isTickTuple(Tuple tuple) {
  return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) && tuple.getSourceStreamId().equals(
      Constants.SYSTEM_TICK_STREAM_ID);
}
 

private static boolean isTickTuple(Tuple tuple) {
    return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID)
            && tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID);
}