backtype.storm.task.TopologyContext Java Examples

@SuppressWarnings("rawtypes")
@Override
void prepare(Map conf, TopologyContext context) {
	// use TTL and maxSize from config if they were not set explicitly using the constructor (implicit way of doing this...)
	if(TTL == 29) TTL = conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC) == null ? TTL : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC)).intValue();
	if(maxSize == 256) maxSize = conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE) == null ? maxSize : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE)).intValue();
	history = new History(this);
	
	// IF NO grouping was set THEN select the first grouping registered for the spout as the grouping used within the Spout (usually a good guess)
	if(groupBy == null){
		Map<GlobalStreamId, Grouping> sources = context.getSources(context.getThisComponentId());
		for(GlobalStreamId id : sources.keySet()){
			Grouping grouping = sources.get(id);
			this.groupBy = new Fields(grouping.get_fields());
			break;
		}
	}
	
	// prepare the selector and operation
	try {
		batcher.prepare(conf);
		operation.prepare(conf, context);
	} catch (Exception e) {
		logger.error("Unable to preapre the Selector or Operation", e);
	}
}
 

public TaskHeartbeatTrigger(Map conf, String name, DisruptorQueue controlQueue, int taskId, String componentId,
                            TopologyContext sysTopologyCtx, ITaskReportErr reportError, TaskStatus executorStatus) {
    this.name = name;
    this.queue = controlQueue;
    this.opCode = TimerConstants.TASK_HEARTBEAT;

    this.taskId = taskId;
    this.componentId = componentId;
    this.sysTopologyCtx = sysTopologyCtx;

    this.frequency = JStormUtils.parseInt(conf.get(Config.TASK_HEARTBEAT_FREQUENCY_SECS), 10);
    this.firstTime = 0;

    this.executeThreadHbTime = TimeUtils.current_time_secs();
    this.taskHbTimeout = JStormUtils.parseInt(conf.get(Config.NIMBUS_TASK_TIMEOUT_SECS), 180);
    this.intervalCheck = new IntervalCheck();
    this.intervalCheck.setInterval(taskHbTimeout);
    this.intervalCheck.start();

    this.reportError = reportError;

    this.uptime = new UptimeComputer();
    this.executorStatus = executorStatus;
}
 

@Override
public void prepare(Map config, TopologyContext context) {
  // Optional set logging
  if (config.get(CONF_LOGGING) != null) {
    m_logging = (Boolean) config.get(CONF_LOGGING);
  } else {
    m_logging = false;
  }

  LOG.info("Loading SVM model...");
  m_dataset = Configuration.getDataSetSemEval2013();
  m_model = SerializationUtils.deserialize(m_dataset.getDatasetPath()
      + File.separator + SVM.SVM_MODEL_FILE_SER);

  if (m_model == null) {
    LOG.error("Could not load SVM model! File: " + m_dataset.getDatasetPath()
        + File.separator + SVM.SVM_MODEL_FILE_SER);
    throw new RuntimeException();
  }
}
 

@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
    this.collector = outputCollector;
    this.client = new EagleServiceClientImpl(config.getConfig());
    if (this.batchSize > 0) {
        this.batchSender = client.batch(this.batchSize);
    }
    this.rackResolver = new DefaultTopologyRackResolver();
    if (config.dataExtractorConfig.resolverCls != null) {
        try {
            rackResolver = config.dataExtractorConfig.resolverCls.newInstance();
            rackResolver.prepare(config);
        } catch (InstantiationException | IllegalAccessException e) {
            LOG.error(e.getMessage(), e);
        }
    }
}
 

@Override
public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
    String[] parts = ((String) registrationArgument).split(":", 2);
    host = parts[0];
    port = Integer.valueOf(parts[1]);
    try {
        socket = new Socket(host, port);
        out = socket.getOutputStream();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public void prepare(Map stormConf, TopologyContext context)	throws Exception {
	fetcher.prepare(stormConf, context);
	if(sio) singleInOp.prepare(stormConf, context);
	else batchOp.prepare(stormConf, context);
	results = new ArrayList<CVParticle>();
}
 

@Override
public void update(org.apache.heron.api.topology.TopologyContext topologyContext) {
  if (delegate instanceof IUpdatable) {
    ((IUpdatable) delegate).update(topologyContext);
  } else {
    LOG.warning(String.format("Update() event received but can not call update() on delegate "
        + "because it does not implement %s: %s", IUpdatable.class.getName(), delegate));
  }
}
 

@Override
public void prepare(org.apache.heron.api.topology.TopologyContext context,
                    String component, String streamId,
                    List<Integer> targetTasks) {
  TopologyContext c = new TopologyContext(context);
  GlobalStreamId g = new GlobalStreamId(component, streamId);
  delegate.prepare(c, g, targetTasks);
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _rand = new Random();
    startTime = System.currentTimeMillis();
    maxSendNum = JStormUtils.parseLong(conf.get("max.send.num"), 1000L);
}
 

@Override
public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
    LOG.info("Preparing Sequence File Bolt...");
    if (this.format == null) throw new IllegalStateException("SequenceFormat must be specified.");

    this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
    this.codecFactory = new CompressionCodecFactory(hdfsConfig);
}
 

@Override
public void prepare(Map stormConf, TopologyContext context) {
  try {
    prepareHDFS();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.conf = conf;
    _collector = collector;
    _rand = new Random();
    sendingCount = 0;
    startTime = System.currentTimeMillis();
    sendNumPerNexttuple = JStormUtils.parseInt(conf.get("send.num.each.time"), 1);
    isStatEnable = JStormUtils.parseBoolean(conf.get("is.stat.enable"), true);
}
 

@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context)	throws Exception {
	images = MjpegStreamingOp.getImages();
	
	ApplicationAdapter connector = new ApplicationAdapter(new MjpegStreamingOp());
	server = HttpServerFactory.create ("http://localhost:"+port+"/", connector);
	server.start();		
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _clients = new ArrayList<>();
    if (_local_drpc_id == null) {
        _backround = new ExtendedThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
                new SynchronousQueue<Runnable>());
        _futures = new LinkedList<>();

        int numTasks = context.getComponentTasks(context.getThisComponentId()).size();
        int index = context.getThisTaskIndex();

        int port = Utils.getInt(conf.get(Config.DRPC_INVOCATIONS_PORT));
        List<String> servers = NetWorkUtils.host2Ip((List<String>) conf.get(Config.DRPC_SERVERS));

        if (servers == null || servers.isEmpty()) {
            throw new RuntimeException("No DRPC servers configured for topology");
        }

        if (numTasks < servers.size()) {
            for (String s : servers) {
                _futures.add(_backround.submit(new Adder(s, port, conf)));
            }
        } else {
            int i = index % servers.size();
            _futures.add(_backround.submit(new Adder(servers.get(i), port, conf)));
        }
    }

}
 

@Test
public void testOpen() throws Exception {
    Map conf = mock(HashMap.class);
    TopologyContext context = mock(TopologyContext.class);
    SpoutOutputCollector collector = mock(SpoutOutputCollector.class);
    when(context.getThisTaskId()).thenReturn(1);
    when(context.getComponentTasks(anyString())).thenReturn(new ArrayList<Integer>() {{
        add(1);
    }});
    HiveJobRunningSourcedStormSpoutProvider provider = new HiveJobRunningSourcedStormSpoutProvider();
    HiveJobFetchSpout spout = (HiveJobFetchSpout)provider.getSpout(config, 1);
    spout.open(conf, context, collector);
    Long yesterday = Long.valueOf(new String(curator.getData().forPath(SHARE_RESOURCES)));
    Assert.assertTrue(System.currentTimeMillis() > yesterday);
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
	this.collector = collector;
	if (this.config == null) {
		config = new KafkaSpoutConfig();
	}
	config.configure(conf);
	zkState = new ZkState(conf, config);
	coordinator = new PartitionCoordinator(conf, config, context, zkState);
	lastUpdateMs = System.currentTimeMillis();
}
 

public static void reportError(TopologyContext topologyContext, String errorMessge) {
    StormClusterState zkCluster = topologyContext.getZkCluster();
    String topologyId = topologyContext.getTopologyId();
    int taskId = topologyContext.getThisTaskId();

    try {
        zkCluster.report_task_error(topologyId, taskId, errorMessge);
    } catch (Exception e) {
        LOG.warn("Failed to report Error");
    }
}
 

@Override
public void prepare(Map config, TopologyContext context) {
  this.m_dataset = Configuration.getDataSetSemEval2013();

  // Optional set logging
  if (config.get(CONF_LOGGING) != null) {
    m_logging = (Boolean) config.get(CONF_LOGGING);
  } else {
    m_logging = false;
  }

  // TODO serialize CombinedFeatureVectorGenerator
  List<FeaturedTweet> featuredTrainTweets = SerializationUtils
      .deserialize(m_dataset.getTrainDataSerializationFile());
  if (featuredTrainTweets != null) {
    TweetTfIdf tweetTfIdf = TweetTfIdf.createFromTaggedTokens(
        FeaturedTweet.getTaggedTokensFromTweets(featuredTrainTweets),
        TfType.LOG, TfIdfNormalization.COS, true);

    LOG.info("Load CombinedFeatureVectorGenerator...");
    m_fvg = new CombinedFeatureVectorGenerator(true, tweetTfIdf);

  } else {
    LOG.error("TaggedTweets could not be found! File is missing: "
        + m_dataset.getTrainDataSerializationFile());
  }
}
 

private Set<GlobalStreamId> getComponentStreams(TopologyContext context) {
    Set<GlobalStreamId> streams = new HashSet<>();
    for (GlobalStreamId streamId : context.getThisSources().keySet()) {
        if (!streamId.get_streamId().equals(CheckpointSpout.CHECKPOINT_STREAM_ID)) {
            streams.add(streamId);
        }
    }
    return streams;
}
 

public void initMonitor(TopologyContext context) {
    isEnableMetrics = true;
    metricClient = new MetricClient(context);
    rocksDbFlushAndCpLatency = metricClient.registerHistogram("RocksDB flush and checkpoint latency");
    hdfsWriteLatency = metricClient.registerHistogram("HDFS write latency");
    hdfsDeleteLatency = metricClient.registerHistogram("HDFS delete latency");
}
 

@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    lastModifiedTracker = new NthLastModifiedTimeTracker(
            deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds));
}
 

public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    this.count = 0l;
    this.isAckRequired = Utils.getBoolean(conf.get("is.ack.required"), false);
    this.sendNumEachTime = Utils.getInt(conf.get("send.num.each.time"), 1);
    this.startTimeStamp = System.currentTimeMillis();
}
 

/**
 * get current task's output <Stream_id, <componentId, MkGrouper>>
 */
public static Map<String, Map<String, MkGrouper>> outbound_components(
        TopologyContext topology_context, WorkerData workerData) {
    Map<String, Map<String, MkGrouper>> rr = new HashMap<>();

    // <Stream_id,<component,Grouping>>
    Map<String, Map<String, Grouping>> output_groupings = topology_context.getThisTargets();
    for (Entry<String, Map<String, Grouping>> entry : output_groupings.entrySet()) {
        String stream_id = entry.getKey();
        Map<String, Grouping> component_grouping = entry.getValue();

        Fields out_fields = topology_context.getThisOutputFields(stream_id);

        Map<String, MkGrouper> componentGrouper = new HashMap<>();
        for (Entry<String, Grouping> cg : component_grouping.entrySet()) {
            String component = cg.getKey();
            Grouping tgrouping = cg.getValue();

            List<Integer> outTasks = topology_context.getComponentTasks(component);
            // ATTENTION: If topology set one component parallelism as 0
            // so we don't need send tuple to it
            if (outTasks.size() > 0) {
                MkGrouper grouper = new MkGrouper(
                        topology_context, out_fields, tgrouping, component, stream_id, workerData);
                componentGrouper.put(component, grouper);
            }
            LOG.info("outbound_components, {}-{} for task-{} on {}",
                    component, outTasks, topology_context.getThisTaskId(), stream_id);
        }
        if (componentGrouper.size() > 0) {
            rr.put(stream_id, componentGrouper);
        }
    }
    return rr;
}
 

public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    
    if (conf.get("spout.max.sending.num") == null) {
        isLimited = false;
    } else {
        isLimited = true;
        SPOUT_MAX_SEND_NUM = JStormUtils.parseLong(conf.get("spout.max.sending.num"));
    }
    
    Boolean btrue = JStormUtils.parseBoolean(conf.get("spout.send.contrl.message"));
    if (btrue != null && btrue) {
        isSendCtrlMsg = true;
    } else {
        isSendCtrlMsg = false;
    }
    
    isFinished = false;
    
    tpsCounter = new TpsCounter(context.getThisComponentId() + ":" + context.getThisTaskId());
    MetricClient metricClient = new MetricClient(context);
    this.tpCounter = metricClient.registerTopologyCounter("TpCounter");

    MAX_PENDING_COUNTER = getMaxPending(conf);
    
    bufferLen = JStormUtils.parseInt(conf.get("byte.buffer.len"), 0);
    
    random = new Random();
    random.setSeed(System.currentTimeMillis());

    idGenerate = new Random(Utils.secureRandomLong());

    JStormUtils.sleepMs(20 * 1000);
    
    LOG.info("Finish open, buffer Len:" + bufferLen);
}
 

@Override
public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
    LOG.info("Preparing Sequence File Bolt...");
    if (this.format == null) throw new IllegalStateException("SequenceFormat must be specified.");

    this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
    this.codecFactory = new CompressionCodecFactory(hdfsConfig);
}
 

private void init(Map stormConf, TopologyContext context, OutputCollector collector,
                  KeyValueState<TaskStream, WindowState> windowState) {
    if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME)) {
        msgIdFieldName = (String) stormConf.get(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME);
    } else {
        throw new IllegalArgumentException(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME + " is not set");
    }
    topologyContext = context;
    outputCollector = collector;
    streamState = windowState;
    pendingTuples = new ArrayList<>();
    recoveryStates = new HashMap<>();
    stateInitialized = false;
    prePrepared = false;
}
 

@SuppressWarnings("rawtypes")
@Override
protected void prepareOpenCVOp(Map stormConf, TopologyContext context) throws Exception { }
 

@Override
public void open(Map conf, TopologyContext context) {
    index = 0;
}
 

void unanchoredSend(TopologyContext topologyContext, TaskSendTargets taskTargets,
                    TaskTransfer transfer_fn, String stream, List<Object> values) {
    UnanchoredSend.send(topologyContext, taskTargets, transfer_fn, stream, values);
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
}