Java Code Examples for org.apache.storm.task.TopologyContext#getThisComponentId()

/**
 * Loads the last saved checkpoint state the from persistent storage.
 */
private KeyValueState<String, CheckPointState> loadCheckpointState(Map<String, Object> conf,
                                                                   TopologyContext ctx) {
  String namespace = ctx.getThisComponentId() + "-" + ctx.getThisTaskId();
  KeyValueState<String, CheckPointState> state
      = (KeyValueState<String, CheckPointState>) StateFactory.getState(namespace, conf, ctx);
  if (state.get(TX_STATE_KEY) == null) {
    CheckPointState txState = new CheckPointState(-1, COMMITTED);
    state.put(TX_STATE_KEY, txState);
    state.commit();
    LOG.fine(() -> String.format("Initialized checkpoint spout state with txState %s", txState));
  } else {
    LOG.fine(() -> String.format("Got checkpoint spout state %s", state.get(TX_STATE_KEY)));
  }
  return state;
}
 

@Override
@SuppressWarnings({ "rawtypes" })
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.componentId = context.getThisComponentId();
    this.spoutId = String.format("%s-%s", componentId, context.getThisTaskId());
    this.collector = collector;
    pendingMessageRetries.clear();
    failedMessages.clear();
    try {
        consumer = createConsumer();
        LOG.info("[{}] Created a pulsar consumer on topic {} to receive messages with subscription {}", spoutId,
                pulsarSpoutConf.getTopic(), pulsarSpoutConf.getSubscriptionName());
    } catch (PulsarClientException e) {
        LOG.error("[{}] Error creating pulsar consumer on topic {}", spoutId, pulsarSpoutConf.getTopic(), e);
        throw new IllegalStateException(format("Failed to initialize consumer for %s-%s : %s",
                pulsarSpoutConf.getTopic(), pulsarSpoutConf.getSubscriptionName(), e.getMessage()), e);
    }
    context.registerMetric(String.format("PulsarSpoutMetrics-%s-%s", componentId, context.getThisTaskIndex()), this,
            pulsarSpoutConf.getMetricsTimeIntervalInSecs());
}
 

@SuppressWarnings({ "rawtypes" })
@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    this.componentId = context.getThisComponentId();
    this.boltId = String.format("%s-%s", componentId, context.getThisTaskId());
    this.collector = collector;
    try {
        sharedPulsarClient = SharedPulsarClient.get(componentId, clientConf);
        producer = sharedPulsarClient.getSharedProducer(producerConf);
        LOG.info("[{}] Created a pulsar producer on topic {} to send messages", boltId, pulsarBoltConf.getTopic());
    } catch (PulsarClientException e) {
        LOG.error("[{}] Error initializing pulsar producer on topic {}", boltId, pulsarBoltConf.getTopic(), e);
        throw new IllegalStateException(
                format("Failed to initialize producer for %s : %s", pulsarBoltConf.getTopic(), e.getMessage()), e);
    }
    context.registerMetric(String.format("PulsarBoltMetrics-%s-%s", componentId, context.getThisTaskIndex()), this,
            pulsarBoltConf.getMetricsTimeIntervalInSecs());
}
 

/**
 * 初始化
 */
@Override
public void prepare(Map stormConf, TopologyContext context,
		OutputCollector collector) {
	this.counters = new HashMap<String, Integer>();
	this.collector = collector;
	this.name = context.getThisComponentId();
	this.id = context.getThisTaskId();
}
 

/**
 * 初始化
 */
@Override
public void prepare(Map stormConf, TopologyContext context,
		OutputCollector collector) {
	this.counters = new HashMap<String, Integer>();
	this.collector = collector;
	this.name = context.getThisComponentId();
	this.id = context.getThisTaskId();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void open(Map conf, TopologyContext context,
        SpoutOutputCollector collector) {

    super.open(conf, context, collector);

    maxDocsPerBucket = ConfUtils.getInt(conf,
            Constants.SQL_MAX_DOCS_BUCKET_PARAM_NAME, 5);

    tableName = ConfUtils.getString(conf,
            Constants.SQL_STATUS_TABLE_PARAM_NAME, "urls");

    maxNumResults = ConfUtils.getInt(conf,
            Constants.SQL_MAXRESULTS_PARAM_NAME, 100);

    try {
        connection = SQLUtil.getConnection(conf);
    } catch (SQLException ex) {
        LOG.error(ex.getMessage(), ex);
        throw new RuntimeException(ex);
    }

    // determine bucket this spout instance will be in charge of
    int totalTasks = context
            .getComponentTasks(context.getThisComponentId()).size();
    if (totalTasks > 1) {
        logIdprefix = "[" + context.getThisComponentId() + " #"
                + context.getThisTaskIndex() + "] ";
        bucketNum = context.getThisTaskIndex();
    }
}
 

/**
 * Handler called when the dynamic spout opens, this method is responsible for creating and setting triggers for
 * handling the spinning up and down of sidelines.
 * @param spout Dynamic spout instance.
 * @param topologyConfig Topology configuration.
 * @param topologyContext Topology context.
 */
@Override
public void onSpoutOpen(
    final DynamicSpout spout,
    final Map topologyConfig,
    final TopologyContext topologyContext
) {
    this.spout = spout;

    createSidelineTriggers();

    Preconditions.checkArgument(
        spoutConfig.containsKey(SidelineConfig.REFRESH_INTERVAL_SECONDS)
        && spoutConfig.get(SidelineConfig.REFRESH_INTERVAL_SECONDS) != null,
        "Configuration value for " + SidelineConfig.REFRESH_INTERVAL_SECONDS + " is required."
    );

    final long refreshIntervalSeconds = ((Number) spoutConfig.get(SidelineConfig.REFRESH_INTERVAL_SECONDS)).longValue();

    final long refreshIntervalMillis = TimeUnit.SECONDS.toMillis(refreshIntervalSeconds);

    // Why not just start the timer at 0? Because we want to block onSpoutOpen() until the first run of loadSidelines()
    loadSidelines();

    // Repeat our sidelines check periodically
    final String threadName = "[" + DynamicSpout.class.getSimpleName() + ":" + getClass().getSimpleName() + "] Timer on "
        + topologyContext.getThisComponentId() + ":" + topologyContext.getThisTaskIndex();

    timer = new Timer(threadName);
    timer.scheduleAtFixedRate(new TimerTask() {
        @Override
        public void run() {
            // Catch this so that it doesn't kill the recurring task
            try {
                loadSidelines();
            } catch (Exception ex) {
                logger.error("Attempting to loadSidelines() failed {}", ex);
            }
        }
    }, refreshIntervalMillis, refreshIntervalMillis);

    for (final SidelineTrigger sidelineTrigger : sidelineTriggers) {
        sidelineTrigger.open(getSpoutConfig());
    }
}
 

/**
 * Open is called once the spout instance has been deployed to the Storm cluster
 * and is ready to get to work.
 *
 * @param topologyConfig The Storm Topology configuration.
 * @param topologyContext The Storm Topology context.
 * @param spoutOutputCollector The output collector to emit tuples via.
 * @throws IllegalStateException if you attempt to open the spout multiple times.
 */
@Override
public void open(Map topologyConfig, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
    if (isOpen) {
        throw new IllegalStateException("This spout has already been opened.");
    }

    // Save references.
    this.topologyContext = topologyContext;
    this.outputCollector = spoutOutputCollector;

    // Ensure a consumer id prefix has been correctly set.
    if (Strings.isNullOrEmpty((String) getSpoutConfigItem(SpoutConfig.VIRTUAL_SPOUT_ID_PREFIX))) {
        throw new IllegalStateException("Missing required configuration: " + SpoutConfig.VIRTUAL_SPOUT_ID_PREFIX);
    }

    // We do not use the getters for things like the metricsRecorder and coordinator here
    // because each of these getters perform a check to see if the spout is open, and it's not yet until we've
    // finished setting all of these things up.

    // Initialize Metric Recorder
    this.metricsRecorder = getFactoryManager().createNewMetricsRecorder();
    this.metricsRecorder.open(getSpoutConfig(), getTopologyContext());

    // Create MessageBuffer
    final MessageBuffer messageBuffer = getFactoryManager().createNewMessageBufferInstance();
    messageBuffer.open(getSpoutConfig());

    // Create MessageBus instance and store into SpoutMessageBus reference reducing accessible scope.
    final MessageBus messageBus = new MessageBus(messageBuffer);
    this.messageBus = messageBus;

    // Define thread context, this allows us to use contextually relevant thread names.
    final ThreadContext threadContext = new ThreadContext(
        topologyContext.getThisComponentId(),
        topologyContext.getThisTaskIndex()
    );

    // Create Coordinator instance and call open.
    spoutCoordinator = new SpoutCoordinator(
        getSpoutConfig(),
        threadContext,
        messageBus,
        metricsRecorder
    );
    spoutCoordinator.open();

    // Define consumer cohort definition.
    final ConsumerPeerContext consumerPeerContext = new ConsumerPeerContext(
        topologyContext.getComponentTasks(topologyContext.getThisComponentId()).size(),
        topologyContext.getThisTaskIndex()
    );

    // TODO: This should be configurable and created dynamically, the problem is that right now we are still tightly
    // coupled to the VirtualSpout implementation.
    this.virtualSpoutFactory = new VirtualSpoutFactory(
        spoutConfig,
        consumerPeerContext,
        factoryManager,
        metricsRecorder
    );

    // Our spout is open, it's not dependent upon the handler to finish opening for us to be 'opened'
    // This is important, because if we waited most of our getters that check the opened state of the
    // spout would throw an exception and make them unusable.
    isOpen = true;

    this.spoutHandler = getFactoryManager().createSpoutHandler();
    this.spoutHandler.open(spoutConfig, virtualSpoutFactory);
    this.spoutHandler.onSpoutOpen(this, topologyConfig, topologyContext);
}
 

@Override
public void open(Map stormConf, TopologyContext context,
        SpoutOutputCollector collector) {

    super.open(stormConf, context, collector);

    indexName = ConfUtils.getString(stormConf, ESStatusIndexNameParamName,
            "status");

    // one ES client per JVM
    synchronized (AbstractSpout.class) {
        try {
            if (client == null) {
                client = ElasticSearchConnection.getClient(stormConf,
                        ESBoltType);
            }
        } catch (Exception e1) {
            LOG.error("Can't connect to ElasticSearch", e1);
            throw new RuntimeException(e1);
        }
    }

    // if more than one instance is used we expect their number to be the
    // same as the number of shards
    int totalTasks = context.getComponentTasks(context.getThisComponentId())
            .size();
    if (totalTasks > 1) {
        logIdprefix = "[" + context.getThisComponentId() + " #"
                + context.getThisTaskIndex() + "] ";

        // determine the number of shards so that we can restrict the
        // search

        // TODO use the admin API when it gets available
        // TODO or the low level one with
        // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-shards-stores.html
        // TODO identify local shards and use those if possible

        // ClusterSearchShardsRequest request = new
        // ClusterSearchShardsRequest(
        // indexName);
        // ClusterSearchShardsResponse shardresponse = client.admin()
        // .cluster().searchShards(request).actionGet();
        // ClusterSearchShardsGroup[] shardgroups =
        // shardresponse.getGroups();
        // if (totalTasks != shardgroups.length) {
        // throw new RuntimeException(
        // "Number of ES spout instances should be the same as number of
        // shards ("
        // + shardgroups.length + ") but is " + totalTasks);
        // }
        // shardID = shardgroups[context.getThisTaskIndex()].getShardId()
        // .getId();

        // TEMPORARY simply use the task index as shard index
        shardID = context.getThisTaskIndex();
        LOG.info("{} assigned shard ID {}", logIdprefix, shardID);
    }

    partitionField = ConfUtils.getString(stormConf,
            ESStatusBucketFieldParamName, "key");

    bucketSortField = ConfUtils.loadListFromConf(
            ESStatusBucketSortFieldParamName, stormConf);

    totalSortField = ConfUtils.getString(stormConf,
            ESStatusGlobalSortFieldParamName);

    maxURLsPerBucket = ConfUtils.getInt(stormConf, ESStatusMaxURLsParamName,
            1);
    maxBucketNum = ConfUtils.getInt(stormConf, ESStatusMaxBucketParamName,
            10);

    queryTimeout = ConfUtils.getInt(stormConf,
            ESStatusQueryTimeoutParamName, -1);

    filterQueries = ConfUtils.loadListFromConf(ESStatusFilterParamName, stormConf);
}