backtype.storm.tuple.Values Java Examples

@Override
public void execute(Tuple input,  BasicOutputCollector collector) {
    Record record = (Record)input.getValueByField(DefaultKinesisRecordScheme.FIELD_RECORD);
    ByteBuffer buffer = record.getData();
    String data = null; 
    try {
        data = decoder.decode(buffer).toString();
        JSONObject jsonObject = new JSONObject(data);

        String referrer = jsonObject.getString("referrer");

        int firstIndex = referrer.indexOf('.');
        int nextIndex = referrer.indexOf('.',firstIndex+1);
        collector.emit(new Values(referrer.substring(firstIndex+1,nextIndex)));

    } catch (CharacterCodingException|JSONException|IllegalStateException e) {
        LOG.error("Exception when decoding record ", e);
    }
}
 

@Override
public void execute(Tuple input, BasicOutputCollector collector) {
    Long tupleId = input.getLong(0);
    Object object = input.getValue(1);

    if (object instanceof TradeCustomer) {

        TradeCustomer tradeCustomer = (TradeCustomer) object;

        Pair trade = tradeCustomer.getTrade();
        Pair customer = tradeCustomer.getCustomer();

        collector.emit(SequenceTopologyDef.TRADE_STREAM_ID, new Values(tupleId, trade));
        collector.emit(SequenceTopologyDef.CUSTOMER_STREAM_ID, new Values(tupleId, customer));
        emitCounter.update(2);
    }
}
 

@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    //Get the word contents from the tuple
    String word = tuple.getString(0);
    //Have we counted any already?
    Integer count = counts.get(word);
    if (count == null)
        count = 0;
    //Increment the count and store it
    count++;
    counts.put(word, count);
    //Emit the word and the current count
    //collector.emit(new Values(IGNITE_FIELD, count));
    TreeMap<String, Integer> words = new TreeMap<>();
    words.put(word,count);

    collector.emit(new Values(words));
    //Log information
    logger.info("Emitting a count of " + count + " for word " + word);
}
 

public void nextTuple() {

    // System.out.println("nextTuple of PcapSimulatorSpout");
    ipAddr.setLength(0);
    String srcAddr = ipAddr.append(randomIpSegmentGenerator.nextInt(255)).append('.').append(randomIpSegmentGenerator.nextInt(255))
        .append('.').append(randomIpSegmentGenerator.nextInt(255)).append('.').append(randomIpSegmentGenerator.nextInt(255)).toString();
    ipAddr.setLength(0);
    String dstAddr = ipAddr.append(randomIpSegmentGenerator.nextInt(255)).append('.').append(randomIpSegmentGenerator.nextInt(255))
        .append('.').append(randomIpSegmentGenerator.nextInt(255)).append('.').append(randomIpSegmentGenerator.nextInt(255)).toString();

    String key = PcapUtils.getSessionKey(srcAddr, dstAddr, String.valueOf(randomProtocolGenerator.nextInt(255)),
        String.valueOf(randomPortGenerator.nextInt(64000)), String.valueOf(randomPortGenerator.nextInt(64000)), "0", "0");

    jsonDoc = jsonDocs[randomJsonGenerator.nextInt(8)];
    ts = System.currentTimeMillis() + randomPortGenerator.nextInt();
    randomPcapGenerator.nextBytes(pcap);

    collector.emit(new Values(srcAddr, key.toString(), jsonDoc, ts, pcap));

    collector.emit("pcap_index_stream", new Values(jsonDoc, key));
    collector.emit("pcap_header_stream", new Values(jsonDoc, key));
    collector.emit("pcap_data_stream", new Values(key.toString(), ts, pcap));

  }
 

@Override
public void execute(Tuple tuple) {
  // We need to ack a tuple when we consider it is done successfully
  // Or we could fail it by invoking collector.fail(tuple)
  // If we do not do the ack or fail explicitly
  // After the MessageTimeout Seconds, which could be set in Config,
  // the spout will fail this tuple
  ++nItems;
  if (nItems % 10000 == 0) {
    long latency = System.currentTimeMillis() - startTime;
    System.out.println("Bolt processed " + nItems + " tuples in " + latency + " ms");
    GlobalMetrics.incr("selected_items");
  }
  if (emit) {
    collector.emit(tuple, new Values(tuple.getString(0) + "!!!"));
  }
  collector.ack(tuple);
}
 

public static StormTopology buildTopology(LocalDRPC drpc) {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3,
            new Values("the cow jumped over the moon"),
            new Values("the man went to the store and bought some candy"),
            new Values("four score and seven years ago"), new Values("how many apples can you eat"),
            new Values("to be or not to be the person"));
    spout.setCycle(true);
    
    TridentTopology topology = new TridentTopology();
    TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16)
            .each(new Fields("sentence"), new Split(), new Fields("word")).groupBy(new Fields("word"))
            .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
            .parallelismHint(16);
            
    topology.newDRPCStream("words", drpc).each(new Fields("args"), new Split(), new Fields("word"))
            .groupBy(new Fields("word"))
            .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
            .each(new Fields("count"), new FilterNull())
            .aggregate(new Fields("count"), new Sum(), new Fields("sum"));
    return topology.build();
}
 

protected void startWatermarkGenerator() {
    LOG.info("Starting watermark generator");
    long watermarkInterval = watermarkGenerator.getWatermarkInterval();
    if (watermarkInterval < MIN_WATERMARK_INTERVAL) {
        throw new IllegalArgumentException("watermark interval must be greater than 10ms!");
    }

    this.timerTriggerThreadPool.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            long newWatermark = watermarkGenerator.getCurrentWatermark();
            if (newWatermark > currentWatermark) {
                LOG.info("Generating new watermark:{}", newWatermark);
                currentWatermark = newWatermark;
                collector.emit(Common.WATERMARK_STREAM_ID, new Values(new Watermark(newWatermark)));

                checkEventTimeWindows();
            }
        }
    }, watermarkInterval, watermarkInterval, TimeUnit.MILLISECONDS);
}
 

public void nextTuple() {
    if (!isLimited) {
        emit();
        return;
    }
    
    if (isFinished) {
        if (isSendCtrlMsg) {
            // LOG.info("spout will send control message due to finish
            // sending ");
            long now = System.currentTimeMillis();
            String ctrlMsg = "spout don't send message due to pending num at " + now;
            collector.emit(SequenceTopologyDef.CONTROL_STREAM_ID, new Values(String.valueOf(now)), ctrlMsg);
        }
        LOG.info("Finish sending ");
        JStormUtils.sleepMs(500);
        return;
    }
    
    if (tupleId > SPOUT_MAX_SEND_NUM) {
        isFinished = true;
        return;
    }
    
    emit();
}
 

@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
	SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");//设置日期格式
	String nowData = df.format(new Date()); // new Date()为获取当前系统时间,检测是否为最新数据

	String data = tuple.getString(0);
	//订单号		用户id	     原金额	                      优惠价	          标示字段		下单时间
	//id		memberid  	totalprice					preprice		sendpay		createdate
	if(data!=null && data.length()>0) {
		String[] values = data.split("\t");
		if(values.length==6) {
			String id = values[0];
			String memberid = values[1];
			String totalprice = values[2];
			String preprice = values[3];
			String sendpay = values[4];
			String createdate = values[5];
			
			if(StringUtils.isNotEmpty(id)&&StringUtils.isNotEmpty(memberid)&&StringUtils.isNotEmpty(totalprice)) {
				if(DateUtils.isValidDate(createdate, nowData)) {
					collector.emit(new Values(id,memberid,totalprice,preprice,sendpay,createdate));
				}
			}
		}
	}
}
 

public void emit() {
    emitCounter.inc();
    Pair trade = PairMaker.makeTradeInstance();
    Pair customer = PairMaker.makeCustomerInstance();

    TradeCustomer tradeCustomer = new TradeCustomer();
    tradeCustomer.setTrade(trade);
    tradeCustomer.setCustomer(customer);
    tradeCustomer.setBuffer(null);

    tradeSumCounter.update(trade.getValue());
    customerSumCounter.update(customer.getValue());

    collector.emit(new Values(idGenerator.nextLong(), tradeCustomer), tupleId);
    tupleId++;
    handleCounter.incrementAndGet();
    while (handleCounter.get() >= MAX_PENDING_COUNTER - 1) {
        JStormUtils.sleepMs(1);
    }
}
 

private void processWatermark(Tuple input) {
    long watermark = ((Watermark) input.getValue(0)).getTimestamp();
    // emit watermark to downstream tasks
    collector.emit(Common.WATERMARK_STREAM_ID, new Values(input.getValue(0)));

    Integer taskId = input.getSourceTask();
    Long taskWatermark = upstreamWatermarks.get(taskId);
    if (taskWatermark == null) {
        upstreamWatermarks.put(taskId, watermark);
    } else if (watermark > taskWatermark) {
        upstreamWatermarks.put(taskId, watermark);
    }

    // todo: needs optimization to update watermark, a bit slow for now
    long minWatermark = Collections.min(upstreamWatermarks.values());
    if (minWatermark > currentWatermark) {
        currentWatermark = minWatermark;
        LOG.debug("Updating current watermark to {}({})",
                currentWatermark, TimeUtils.format((int) (currentWatermark / 1000L)));
    }

    checkEventTimeWindows();
}
 

@Override
public void execute(Tuple input) {
    if (input == null) {
        return;
    }
    DataSource dataSource = (DataSource) input.getValueByField(HadoopClusterConstants.FIELD_DATASOURCE);
    DataType dataType = (DataType) input.getValueByField(HadoopClusterConstants.FIELD_DATATYPE);
    Object data = input.getValueByField(HadoopClusterConstants.FIELD_DATA);

    List<TaggedLogAPIEntity> entities = (List<TaggedLogAPIEntity>) data;
    if (dataType.equals(DataType.METRIC)) {
        writeEntities(entities, dataType, dataSource);
    } else {
        for (TaggedLogAPIEntity entity : entities) {
            if (entity instanceof RunningQueueAPIEntity) {
                RunningQueueAPIEntity queue = (RunningQueueAPIEntity) entity;
                if (queue.getUsers() != null && !queue.getUsers().getUsers().isEmpty() && queue.getMemory() != 0) {
                    String queueName = queue.getTags().get(HadoopClusterConstants.TAG_QUEUE);
                    collector.emit(new Values(queueName, QueueStreamInfo.convertEntityToStream(queue)));
                }
            }
        }
        writeEntities(entities, dataType, dataSource);
    }
    this.collector.ack(input);
}
 

@Override
@SuppressWarnings("rawtypes")
public List<Object> deserialize(byte[] ser) {
    try {
        if (ser != null) {
            Map map = mapper.readValue(ser, Map.class);
            Object message = map.get(MESSAGE_SCHEME_KEY);
            if (message != null) {
                return new Values(map.get(MESSAGE_SCHEME_KEY));
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Content is null, ignore");
            }
        }
    } catch (IOException e) {
        try {
            LOG.error("Failed to deserialize as JSON: {}", new String(ser, "UTF-8"), e);
        } catch (Exception ex) {
            LOG.error(ex.getMessage(), ex);
        }
    }
    return null;
}
 

private boolean finishBatch(TrackedBatch tracked, Tuple finishTuple) {
    boolean success = true;
    try {
        _bolt.finishBatch(tracked.info);
        String stream = COORD_STREAM(tracked.info.batchGroup);
        _collector.flush();
        for (Integer task : tracked.condition.targetTasks) {
            _collector.emitDirect(task, stream, finishTuple, new Values(tracked.info.batchId,
                    Utils.get(tracked.taskEmittedTuples, task, 0)));
        }
        if (tracked.delayedAck != null) {
            _collector.ack(tracked.delayedAck);
            tracked.delayedAck = null;
        }
    } catch (FailedException e) {
        failBatch(tracked, e);
        success = false;
    }
    _batches.remove(tracked.info.batchId.getId());
    return success;
}
 

@Override
public void nextTuple() {
    if (index >= WORDS.size()) {
        JStormUtils.sleepMs(10);
        //collector.emit(Common.WATERMARK_STREAM_ID, new Values(new Watermark(15L)));
        return;
    }

    String[] words = WORDS.get(index);
    long[] times = TIMES.get(index);
    for (int i = 0; i < words.length; i++) {
        LOG.info("emitting tuple, word:{}, event time:{}", words[i], TIME_BASE + times[i]);
        collector.emit(new Values(words[i], TIME_BASE + times[i]));
    }

    index++;
    JStormUtils.sleepMs(5);
}
 

public void serializeInto(List<Object> values, Output out) throws IOException {
	if (values instanceof Values) {
		Values val = (Values) values;
        out.writeInt(val.type, true);
	    switch (val.type) {
	        case Values.OBJECT:
	        	serialize(values, out);
	        	break;
	        case Values.STRING:
	        	serializeStrings(values, out);
	        	break;
	        case Values.INTEGER:
	        	serializeIntegers(values, out);
	        	break;
	    }
	} else {
		out.writeInt(Values.OBJECT, true);
		serialize(values, out);
	}
}
 

/**
 * assumes that batchId starts at 0, and is increasing
 *
 * @param batchId
 * @param collector
 */
@Override
public void emitBatch(long batchId, TridentCollector collector) {
	long id = (batchId-1) * this.batchSize;
	for(int i = 0; i < this.batchSize; i++) {
		if(id % latFreq == nextlat) {
			time = System.currentTimeMillis();
			if(--nextlat <= 0) {
				nextlat = 1000;
			}
		}
		collector.emit(new Values(id, this.host, this.time, this.payload));
		time = 0;
		id++;
	}
}
 

@SuppressWarnings("rawtypes")
@Override
public void finishBatch() {
	try{
		if(resultOp != null){
			String result = resultOp.getFinalResult();
			collector.emit(new Values(requestId, result));
		}else if(batchOp != null){
			List<? extends CVParticle> results = batchOp.getBatchResult();
			for(CVParticle output : results){
				CVParticleSerializer serializer = serializers.get(output.getClass().getName());
				if(serializers.containsKey(output.getClass().getName())){
					collector.emit(serializer.toTuple(output));
				}else{
					// use serializer from operation as fall back
					collector.emit(batchOp.getSerializer().toTuple(output));
				}
			}
		}
	}catch(Exception e){
		logger.error("Unable to finish batch!");
		collector.reportError(e);
	}
}
 

@Override
public void nextTuple() {
	if(delay > 0) {
		if(id % sleepFreq == 0) {
			try { Thread.sleep(delay); } catch (InterruptedException e) { e.printStackTrace();}
		}
	}
	if(id % latFreq == nextlat) {
		time = System.currentTimeMillis();
		if(--nextlat <= 0) {
			nextlat = 1000;
		}
	}

	if(withFt) {
		spoutOutputCollector.emit(new Values(this.id, host, this.time, this.payload), this.id);
	} else {
		spoutOutputCollector.emit(new Values(this.id, host, this.time, this.payload));
	}

	time = 0;
	this.id++;
}
 

/** {@inheritDoc} */

public void execute(Tuple input) {
  try {
    this.connector.getTable().put(conf.getPutFromTuple(input));
  } catch (IOException ex) {

		JSONObject error = ErrorGenerator.generateErrorMessage(
				"Alerts problem: " + input.getBinary(0), ex);
		collector.emit("error", new Values(error));
		
    throw new RuntimeException(ex);
  }

  if (this.autoAck) {
    this.collector.ack(input);
  }
}
 

/**
 * {@inheritDoc}
 */
@Override
public void emit(Object[] data)
    throws StreamingException
{
    /**
     * 目前暂时不支持acker
     */

    if (streamName == null)
    {
        outputCollector.emit(new Values(data));
    }
    else
    {
        outputCollector.emit(streamName, new Values(data));
    }

}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    for(String word: tuple.getString(0).split(" ")) {
        if(word.length() > 0) {
            collector.emit(new Values(word));
        }
    }
}
 

@Override
public List<Object> deserializeKeyAndValue(byte[] key, byte[] value) {
    if ( key == null ) {
        return deserialize(value);
    }
    String keyString = StringScheme.deserializeString(key);
    String valueString = StringScheme.deserializeString(value);
    return new Values(ImmutableMap.of(keyString, valueString));
}
 

@Test
public void shouldEmitFirstFieldOfTuple() {
  bolt.execute(tuple, collector);

  verify(tuple, times(1)).getValue(0);
  verify(collector, times(1)).emit(any(Values.class));
}
 

@Test(dataProvider = "getAnyFields")
public void shouldEmitOnExecute(Item field1, Item field2) {
  PageViewBolt bolt = new PageViewBolt(field1, field2);
  BasicOutputCollector collector = mock(BasicOutputCollector.class);
  Tuple input = MockTupleHelpers.mockAnyTuple();
  when(input.getString(0)).thenReturn("http://view1\t200\t100000\t100");
  bolt.execute(input, collector);
  verify(collector, times(1)).emit(any(Values.class));
}
 

public void execute(Tuple tuple) {
    String sentence = tuple.getStringByField("sentence");
    String[] words = sentence.split(" ");
    for(String word : words){
        this.collector.emit(new Values(word));
    }
}
 

@Override
public void nextTuple() {
    if(content >= limit) {
        JStormUtils.sleepMs(1000);
        return;
    }

    Values values = new Values(task, content);
    collector.emit(values, "msg");
    content++;
    emitCounter.inc();
    LOG.info("nextTuple. task = " + task + " content = " + content);
}
 

@Override
public void nextTuple() {
  if (entranceProcessor.hasNext()) {
    Values value = newValues(entranceProcessor.nextEvent());
    collector.emit(outputStream.getOutputId(), value);
  } else
    Utils.sleep(1000);
  // StormTupleInfo tupleInfo = tupleInfoQueue.poll(50,
  // TimeUnit.MILLISECONDS);
  // if (tupleInfo != null) {
  // Values value = newValues(tupleInfo.getContentEvent());
  // collector.emit(tupleInfo.getStormStream().getOutputId(), value);
  // }
}
 

@Override
public void nextTuple() {
    String sentence = CHOICES[index];
    collector.emit(new Values(sentence), ++msgId);
    index = (++index) % CHOICES.length;
    printSendTps(1);
}
 

public void execute(Tuple input) {
    Integer sourceId = input.getInteger(0);
    Long eventId = input.getLong(1);
    Long recentEvent = recentEventId.get(sourceId);

    if (null != recentEvent && eventId <= recentEvent) {
        String error = "Error: event id is not in strict order! event source Id: " + sourceId +
                ", last event Id: " + recentEvent + ", current event Id: " + eventId;
        _collector.emit(input, new Values(error));
    }
    recentEventId.put(sourceId, eventId);

    _collector.ack(input);
}