org.apache.storm.tuple.Tuple Java Examples

@Override
public void execute(Tuple input) {
    if (TupleUtils.isTick(input)) {
        collector.ack(input);
        return;
    }
    try {
        Command cmd = (Command) input.getValueByField(EmitFields.COMMAND);
        BoltCommandHandler handler = handlerManager.getHandler(cmd);
        handler.handle(input);
        this.collector.ack(input);
    } catch (Exception e) {
        this.collector.fail(input);
        this.collector.reportError(e);
        logger.error("Process Error!", e);
    }

}
 

@Test
public void testQueryNotDoneAndThenDone() {
    bolt = ComponentUtils.prepare(new DonableFilterBolt(), collector);

    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA);
    bolt.execute(query);

    BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord();
    Tuple matching = makeRecordTuple(record);
    bolt.execute(matching);

    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    bolt.execute(tick);

    Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record);
    Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected));

    BulletRecord anotherRecord = RecordBox.get().add("field", "b235gf23b").add("mid", "2342").getRecord();
    Tuple anotherExpected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", anotherRecord);
    Assert.assertFalse(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, anotherExpected));
}
 

@Override
public void handle(Tuple tuple) {
    EmitData data = (EmitData) tuple.getValueByField(Constants.EmitFields.DATA);
    List<PairWrapper<String, Object>> wrapperList = data.get(EmitData.MESSAGE);
    if (wrapperList != null && !wrapperList.isEmpty()) {
        for (PairWrapper<String, Object> wrapper : wrapperList) {
            HeartbeatPulse pulse = HeartbeatPulse.build(wrapper.pairs2map());
            if (logger.isDebugEnabled()) {
                Object offset = data.get(EmitData.OFFSET);
                HeartBeatPacket packet = HeartBeatPacket.parse(pulse.getPacket());
                SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
                String groupId = tuple.getStringByField(Constants.EmitFields.GROUP_FIELD);
                logger.debug("[heartbeat] {} offset:{} ts:{}, time:{}", groupId, offset == null ? -1 : offset, packet.getTxtime(), format.format(new Date(packet.getTxtime())));
            }
            reporter.mark(pulse);
        }
    }

    handler.handle(tuple);
    this.listener.getOutputCollector().ack(tuple);
}
 

@Override
protected Object extractField(Object target) {
    List<String> fieldNames = getFieldNames();
    for (int i = 0; i < fieldNames.size(); i++) {
        String field = fieldNames.get(i);
        if (target instanceof Tuple) {
            target = ((Tuple) target).getValueByField(field);
            if (target == null) {
                return NOT_FOUND;
            }
        }
        else {
            return NOT_FOUND;
        }
    }
    return target;
}
 

private TriggerPolicy<Tuple, ?> getTriggerPolicy(Count slidingIntervalCount,
                                                 Duration slidingIntervalDuration,
                                                 WindowManager<Tuple> manager,
                                                 EvictionPolicy<Tuple, ?> evicPolicy) {
  if (slidingIntervalCount != null) {
    if (isTupleTs()) {
      return new WatermarkCountTriggerPolicy<>(slidingIntervalCount.value,
          manager, evicPolicy, manager);
    } else {
      return new CountTriggerPolicy<>(slidingIntervalCount.value, manager, evicPolicy);
    }
  } else {
    if (isTupleTs()) {
      return new WatermarkTimeTriggerPolicy<>(slidingIntervalDuration.value,
          manager, evicPolicy, manager);
    } else {
      return new TimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evicPolicy);
    }
  }
}
 

@Override
public void execute(Tuple tuple) {
    List<Object> id = tuple.select(_idFields);
    GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId());
    if (!_pending.containsKey(id)) {
        _pending.put(id, new HashMap<GlobalStreamId, Tuple>());
    }
    Map<GlobalStreamId, Tuple> parts = _pending.get(id);
    if (parts.containsKey(streamId)) {
        throw new RuntimeException("Received same side of single join twice");
    }
    parts.put(streamId, tuple);
    if (parts.size() == _numSources) {
        _pending.remove(id);
        List<Object> joinResult = new ArrayList<Object>();
        for (String outField : _outFields) {
            GlobalStreamId loc = _fieldLocations.get(outField);
            joinResult.add(parts.get(loc).getValueByField(outField));
        }
        _collector.emit(new ArrayList<Tuple>(parts.values()), joinResult);

        for (Tuple part : parts.values()) {
            _collector.ack(part);
        }
    }
}
 

/**
 * The bolt should extract a message and timestamp from a tuple and
 * pass that to a {@code MessageDistributor}.
 */
@Test
public void testExtractMessage() throws Exception {

  ProfileBuilderBolt bolt = createBolt();

  // create a mock
  MessageDistributor distributor = mock(MessageDistributor.class);
  bolt.withMessageDistributor(distributor);

  // create a tuple
  final long timestamp1 = 100000000L;
  Tuple tuple1 = createTuple("entity1", message1, profile1, timestamp1);

  // execute the bolt
  TupleWindow tupleWindow = createWindow(tuple1);
  bolt.execute(tupleWindow);

  // the message should have been extracted from the tuple and passed to the MessageDistributor
  verify(distributor).distribute(any(MessageRoute.class), any());
}
 

@Test
public void testResetTimeout() throws Exception {
    setupExpectationsForTuple();
    setupExpectationsForTopologyContextNoEmit();

    EventCorrelatingOutputCollector sut = getSystemUnderTest();

    Tuple anchor = new TupleImpl(mockedTopologyContext, new Values(PARENT_STREAMLINE_EVENT), TASK_0,
            Utils.DEFAULT_STREAM_ID);

    sut.resetTimeout(anchor);

    new Verifications() {{
        mockedOutputCollector.resetTimeout(anchor); times = 1;
    }};
}
 

@Override
public void execute(Tuple tuple) {
    String url = tuple.getStringByField("url");
    byte[] content = tuple.getBinaryByField("content");
    Metadata metadata = (Metadata) tuple.getValueByField("metadata");
    String text = tuple.getStringByField("text");

    Values v = new Values(url, content, metadata, text);

    // if there is a text - no need to parse it again
    if (StringUtils.isNotBlank(text)) {
        collector.emit(tuple, v);
    } else {
        collector.emit("tika", tuple, v);
    }

    collector.ack(tuple);
}
 

@Override
public void execute(Tuple tuple) {
  Object obj = tuple.getValue(0);
  long count = tuple.getLong(1);
  int source = tuple.getSourceTask();
  Map<Integer, Long> subCounts = counts.get(obj);
  if (subCounts == null) {
    subCounts = new HashMap<Integer, Long>();
    counts.put(obj, subCounts);
  }
  //Update the current count for this object
  subCounts.put(source, count);
  //Output the sum of all the known counts so for this key
  long sum = 0;
  for (Long val: subCounts.values()) {
    sum += val;
  }
  collector.emit(new Values(obj, sum));
}
 

@Override
public boolean mark(Tuple tuple, long offset) {
    // check based on time first
    if (interval != -1) {
        long now = System.currentTimeMillis();
        if (now >= timeStarted + interval) {
            LOG.info(
                    "Rotating file based on time : started {} interval {}",
                    timeStarted, interval);
            return true;
        }
    }

    long diff = offset - this.lastOffset;
    this.currentBytesWritten += diff;
    this.lastOffset = offset;
    boolean size = this.currentBytesWritten >= this.maxBytes;
    if (size) {
        LOG.info(
                "Rotating file based on size : currentBytesWritten {} maxBytes {}",
                currentBytesWritten, maxBytes);
    }
    return size;
}
 

/**
 * *bolt*从单词文件接收到文本行，并标准化它。 文本行会全部转化成小写，并切分它，从中得到所有单词。
 */
public void execute(Tuple input) {
	System.out.println("WordNormalizer.execute()执行次数:"+count);
	String sentence = input.getString(0);
	String[] words = sentence.split(" ");
	for (String word : words) {
		word = word.trim();
		if (!word.isEmpty()) {
			word = word.toLowerCase();
			/* //发布这个单词 */
			collector.emit(input, new Values(word));
		}
	}
	// 对元组做出应答
	collector.ack(input);
	count++;
}
 

@Override
protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple)
        throws IOException {
    AbstractHDFSWriter writer = super.makeNewWriter(path, tuple);

    Instant now = Instant.now();

    // overrides the filename and creation date in the headers
    header_fields.put("WARC-Date", WARCRecordFormat.WARC_DF.format(now));
    header_fields.put("WARC-Filename", path.getName());

    byte[] header = WARCRecordFormat.generateWARCInfo(header_fields);

    // write the header at the beginning of the file
    if (header != null && header.length > 0) {
        super.out.write(Utils.gzip(header));
    }

    return writer;
}
 

/**
 * Saves an operation for later.
 * @param tuple Contains the data elements that need written to HBase.
 */
private void save(Tuple tuple) {
  byte[] rowKey = mapper.rowKey(tuple);
  ColumnList cols = mapper.columns(tuple);
  Durability durability = writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL;

  Optional<Long> ttl = mapper.getTTL(tuple);
  if(ttl.isPresent()) {
    hbaseClient.addMutation(rowKey, cols, durability, ttl.get());
  } else {
    hbaseClient.addMutation(rowKey, cols, durability);
  }

  batchHelper.addBatch(tuple);
  LOG.debug("Added mutation to the batch; size={}", batchHelper.getBatchSize());
}
 

@Test
public void testCountBasedWindowWithGroupby() throws Exception {
    Assert.assertTrue(doTest(readFile("/window-rule-count-withgroupby.json"), 2));
    new Verifications() {
        {
            String streamId;
            Collection<Tuple> anchors;
            List<List<Object>> tuples = new ArrayList<>();
            mockCollector.emit(streamId = withCapture(), anchors = withCapture(), withCapture(tuples));
            Map<String, Object> fieldsAndValues1 = ((StreamlineEvent) tuples.get(0).get(0));
            Assert.assertEquals("count is 2, min salary is 30, max salary is 40", fieldsAndValues1.get("body"));
            Map<String, Object> fieldsAndValues2 = ((StreamlineEvent) tuples.get(1).get(0));
            Assert.assertEquals("count is 5, min salary is 50, max salary is 90", fieldsAndValues2.get("body"));
            Map<String, Object> fieldsAndValues3 = ((StreamlineEvent) tuples.get(2).get(0));
            Assert.assertEquals("count is 1, min salary is 100, max salary is 100", fieldsAndValues3.get("body"));
            Map<String, Object> fieldsAndValues4 = ((StreamlineEvent) tuples.get(3).get(0));
            Assert.assertEquals("count is 4, min salary is 110, max salary is 140", fieldsAndValues4.get("body"));
            Map<String, Object> fieldsAndValues5 = ((StreamlineEvent) tuples.get(4).get(0));
            Assert.assertEquals("count is 5, min salary is 150, max salary is 190", fieldsAndValues5.get("body"));
            Map<String, Object> fieldsAndValues6 = ((StreamlineEvent) tuples.get(5).get(0));
            Assert.assertEquals("count is 1, min salary is 200, max salary is 200", fieldsAndValues6.get("body"));
        }
    };
}
 

@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String word = tuple.getString(0);
    // Thread Polling every 5 seconds to update the wordSet seconds which is
    // used in FilterWords bolt to filter the words
    try {
        if (!poll) {
            wordSet = parseFile(fileName);
            pollTime = System.currentTimeMillis();
            poll = true;
        } else {
            if ((System.currentTimeMillis() - pollTime) > 5000) {
                wordSet = parseFile(fileName);
                pollTime = System.currentTimeMillis();
            }
        }
    } catch (IOException exp) {
        throw new RuntimeException(exp);
    }
    if (wordSet != null && !wordSet.contains(word)) {
        collector.emit(new Values(word));
    }
}
 

@Override
public void execute(Tuple tuple) {
    if (TupleUtils.isTick(tuple)) {
        // if we have a tick tuple then lets see if enough time has passed since our last batch was processed
        if ((System.currentTimeMillis() / 1000 - lastBatchProcessTimeSeconds) >= batchIntervalInSec) {
            LOGGER.debug("Received tick tuple and reached batch interval, executing batch");
            finishBatch();
        } else {
            LOGGER.debug("Received tick tuple, but haven't reached batch interval, nothing to do");
        }
    } else {
        // for a regular tuple we add it to the queue and then see if our queue size exceeds batch size
        this.queue.add(tuple);

        int queueSize = this.queue.size();
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Current queue size is " + queueSize + ", and batch size is " + batchSize);
        }

        if (queueSize >= batchSize) {
            LOGGER.debug("Queue Size is greater than or equal to batch size, executing batch");
            finishBatch();
        }
    }
}
 

@Override
public void execute(Tuple tuple) {
    _rootLogger.debug("root: This is a DEBUG message");
    _rootLogger.info("root: This is an INFO message");
    _rootLogger.warn("root: This is a WARN message");
    _rootLogger.error("root: This is an ERROR message");

    _logger.debug("myapp: This is a DEBUG message");
    _logger.info("myapp: This is an INFO message");
    _logger.warn("myapp: This is a WARN message");
    _logger.error("myapp: This is an ERROR message");

    _subLogger.debug("myapp.sub: This is a DEBUG message");
    _subLogger.info("myapp.sub: This is an INFO message");
    _subLogger.warn("myapp.sub: This is a WARN message");
    _subLogger.error("myapp.sub: This is an ERROR message");

    _collector.emit(tuple, new Values(tuple.getString(0) + "!!!"));
    _collector.ack(tuple);
}
 

@Test
public void testTimeBasedWindowEmptyCondition() throws Exception {
    Assert.assertTrue(doTest(readFile("/window-rule-empty-condition.json"), 1));
    new Verifications() {
        {
            String streamId;
            Collection<Tuple> anchors;
            List<List<Object>> tuples = new ArrayList<>();
            mockCollector.emit(streamId = withCapture(), anchors = withCapture(), withCapture(tuples));
            Assert.assertEquals("outputstream", streamId);
            Map<String, Object> fieldsAndValues1 = ((StreamlineEvent) tuples.get(0).get(0));
            Assert.assertEquals(0, fieldsAndValues1.get("deptid"));
            Assert.assertEquals(40, fieldsAndValues1.get("salary_MAX"));
        }
    };
}
 

@Test
public void shouldEmitNothingIfNormalTupleIsReceived() {
    // given
    Tuple normalTuple = mockRankableTuple(ANY_OBJECT, ANY_COUNT);
    BasicOutputCollector collector = mock(BasicOutputCollector.class);
    IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();

    // when
    bolt.execute(normalTuple, collector);

    // then
    verifyZeroInteractions(collector);
}
 

@Override
protected void process(Tuple tuple) {
    Notification notification = new StreamlineEventAdapter((StreamlineEvent) tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT));
    notificationContext.track(notification.getId(), tuple);
    // send to notifier
    notificationService.notify(notificationSink.getNotifierName(), notification);
}
 

@Override
public void afterBulk(long executionId, BulkRequest request,
        Throwable throwable) {
    eventCounter.scope("bulks_received").incrBy(1);
    LOG.error("Exception with bulk {} - failing the whole lot ",
            executionId, throwable);
    synchronized (waitAck) {
        // WHOLE BULK FAILED
        // mark all the docs as fail
        Iterator<DocWriteRequest<?>> itreq = request.requests().iterator();
        while (itreq.hasNext()) {
            DocWriteRequest bir = itreq.next();
            String id = bir.id();
            List<Tuple> xx = waitAck.getIfPresent(id);
            if (xx != null) {
                LOG.debug("Failed {} tuple(s) for ID {}", xx.size(), id);
                for (Tuple x : xx) {
                    // fail it
                    _collector.fail(x);
                }
                waitAck.invalidate(id);
            } else {
                LOG.warn("Could not find unacked tuple for {}", id);
            }
        }
    }
}
 

@Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
    //The input is a tuple
    Key currentKey = (Key) tuple.getValue(0);
    if (!slicingWindowOperatorMap.containsKey(currentKey)) {
        slicingWindowOperatorMap.put(currentKey, initWindowOperator());
    }
    SlicingWindowOperator<Value> slicingWindowOperator = slicingWindowOperatorMap.get(currentKey);
    //We only process the Value of a tuple
    slicingWindowOperator.processElement((Value) tuple.getValue(1), tuple.getLong(2));
    processWatermark(currentKey, tuple.getLong(2), basicOutputCollector);
}
 

@Test
public void defaultJSONFromPositionShouldReturnJSON() {
  Tuple tuple = mock(Tuple.class);
  when(tuple.getBinary(0)).thenReturn("{\"field\":\"value\"}".getBytes(UTF_8));

  JSONObject expected = new JSONObject();
  expected.put("field", "value");
  MessageGetStrategy messageGetStrategy = MessageGetters.DEFAULT_JSON_FROM_POSITION.get();
  assertEquals(expected, messageGetStrategy.get(tuple));
}
 

private void sendTermination(MetaVersion ver, Tuple input, long offset) {
    DbusMessage message = BoltCommandHandlerHelper.buildTerminationMessage(ver.getSchema(), ver.getTable(), ver.getVersion());

    EmitData data = new EmitData();
    data.add(EmitData.AVRO_SCHEMA, EmitData.NO_VALUE);
    data.add(EmitData.VERSION, ver);
    data.add(EmitData.MESSAGE, message);
    data.add(EmitData.OFFSET, offset);

    this.emit(listener.getOutputCollector(), input, groupField(ver.getSchema(), ver.getTable()), data, Command.DATA_INCREMENT_TERMINATION);

    // 修改data table表状态
    //BoltCommandHandlerHelper.changeDataTableStatus(ver.getSchema(), ver.getTable(), DataTable.STATUS_ABORT);
}
 

private static Tuple pushInto(Tuple mocked, Object... contents) {
    when(mocked.getValues()).thenReturn(Arrays.asList(contents));
    when(mocked.size()).thenReturn(contents.length);
    for (int i = 0; i < contents.length; ++i) {
        when(mocked.getValue(i)).thenReturn(contents[i]);
        when(mocked.getString(i)).thenReturn(Objects.toString(contents[i]));
    }
    return mocked;
}
 

@Override
public void execute(Tuple input, BasicOutputCollector basicOutputCollector) {
    Rankings rankings = (Rankings) input.getValueByField("rankings");
    /**
     * TODO:此处2个步骤的操作应该合并成一个lua操作，不过考虑到更新频率低，并且设置了globalGrouping，已经不存在并发状况了
     */
    hashOperations.getOperations().delete(WORD_COUNT_TOP_N_REAL_TIME_KEY);
    rankings.getRankings().forEach(rankable -> {
        String word = (String) rankable.getObject();
        long count = rankable.getCount();
        hashOperations.put(WORD_COUNT_TOP_N_REAL_TIME_KEY, word, count);
    });

}
 

public void testAlarmDefinitionCreatedEvent() {
  final Map<String, AlarmSubExpression> expressions = createAlarmSubExpressionMap(alarm);
  final AlarmDefinitionCreatedEvent event =
      new AlarmDefinitionCreatedEvent(alarmDefinition.getTenantId(), alarmDefinition.getId(),
          alarmDefinition.getName(), alarmDefinition.getDescription(), alarmDefinition
              .getAlarmExpression().getExpression(), expressions, Arrays.asList("hostname"));
  final Tuple tuple = createTuple(event);
  bolt.execute(tuple);
  verify(collector, times(1)).ack(tuple);
  verify(collector, times(1)).emit(
      EventProcessingBolt.ALARM_DEFINITION_EVENT_STREAM_ID,
      new Values(EventProcessingBolt.CREATED, event));
}
 

public void testAlarmDefinitionDeletedEvent() {
  final AlarmDefinitionDeletedEvent event = createAlarmDefinitionDeletedEvent(alarmDefinition);
  final Tuple tuple = createTuple(event);
  bolt.execute(tuple);
  verify(collector, times(1)).ack(tuple);
  verify(collector, times(1)).emit(
      EventProcessingBolt.ALARM_DEFINITION_EVENT_STREAM_ID,
      new Values(EventProcessingBolt.DELETED, event));
}
 

@Override
public void handle(Tuple tuple) {
    EmitData emitData = (EmitData) tuple.getValueByField(Constants.EmitFields.DATA);
    ControlMessage message = emitData.get(EmitData.MESSAGE);
    Object obj = cache.getIfPresent(message.getId());
    if (obj != null) {
        return;
    }
    cache.put(message.getId(), new Object());
    handler.handle(tuple);
}