Java Code Examples for org.apache.storm.tuple.Tuple#getValueByField()

@Override
public byte[] format(Tuple tuple) {
    if (fields==null || fields.size()==0) {
        throw new IllegalArgumentException("Output field names not specified. Set them using withFields().");
    }

    StreamlineEvent event = ((StreamlineEvent) tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT));
    StringBuilder sb = new StringBuilder();

    for(int i = 0; i < fields.size(); i++) {
        Object value = event.get(fields.get(i));
        if (value!=null) {
            sb.append( value );
        }
        if ( i != fields.size()-1 ) {
            sb.append(this.fieldDelimiter);
        }
    }
    sb.append(this.recordDelimiter);
    return sb.toString().getBytes();
}
 

/**
 * Transfers data into grid.
 *
 * @param tuple Storm tuple.
 */
@SuppressWarnings("unchecked")
@Override public void execute(Tuple tuple) {
    if (stopped)
        return;

    if (!(tuple.getValueByField(igniteTupleField) instanceof Map))
        throw new IgniteException("Map as a streamer input is expected!");

    final Map<K, V> gridVals = (Map<K, V>)tuple.getValueByField(igniteTupleField);

    try {
        if (log.isDebugEnabled())
            log.debug("Tuple (id:" + tuple.getMessageId() + ") from storm: " + gridVals);

        getStreamer().addData(gridVals);

        collector.ack(tuple);
    }
    catch (Exception e) {
        log.error("Error while processing tuple of " + gridVals, e);
        collector.fail(tuple);
    }
}
 

@Override
public void execute(Tuple input) {
    if (TupleUtils.isTick(input)) {
        collector.ack(input);
        return;
    }

    try {
        Command cmd = (Command) input.getValueByField(EmitFields.COMMAND);
        BoltCommandHandler handler = handlerManager.getHandler(cmd);
        handler.handle(input);
        this.collector.ack(input);
    } catch (Exception e) {
        logger.error("Process data error", e);
        this.collector.reportError(e);
        this.collector.fail(input);
    }

}
 

@Override
public void handle(Tuple tuple) {
    try {
        EmitData emitData = (EmitData) tuple.getValueByField(Constants.EmitFields.DATA);
        Command cmd = (Command) tuple.getValueByField(Constants.EmitFields.COMMAND);
        DBusConsumerRecord<String, byte[]> record = emitData.get(EmitData.MESSAGE);
        String schemaStr = new String(record.value(), "utf-8");
        logger.info("Receive Schema:" + Utils.replaceBlanks(schemaStr));

        Map<String, Object> map = (Map<String, Object>) JSON.parse(schemaStr);
        String tableName = map.get(Constants.MessageBodyKey.TABLE_NAME).toString();
        String namespace = map.get(Constants.MessageBodyKey.NAMESPACE).toString();

        this.emit(listener.getOutputCollector(), tuple, Joiner.on(".").join(namespace, tableName), emitData, cmd);
    } catch (Exception e) {
        logger.error("Error when process tuple", JSON.toJSONString(tuple));
        throw new RuntimeException(e);
    }
}
 

@Override
public void execute(Tuple tuple) {
    String url = tuple.getStringByField("url");
    Metadata metadata = (Metadata) tuple.getValueByField("metadata");

    // keep it simple for now and ignore cases where the canonical URL was
    // used
    String sha256hex = org.apache.commons.codec.digest.DigestUtils
            .sha256Hex(url);
    DeleteRequest dr = new DeleteRequest(getIndexName(metadata), sha256hex);
    try {
        client.delete(dr, RequestOptions.DEFAULT);
    } catch (IOException e) {
        _collector.fail(tuple);
        LOG.error("Exception caught while deleting", e);
        return;
    }
    _collector.ack(tuple);
}
 

@Override
public byte[] mapRecord(Tuple tuple) {
    StreamlineEvent event = (StreamlineEvent) tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT);
    StringBuilder builder = new StringBuilder();
    for (String field : this.fields) {
        Object val = event.get(field);
        if (val == null) {
            throw new IllegalArgumentException(String.format("Field '%s' value is missing in the streamline event", field));
        }
        builder.append(val);
        builder.append(fieldDelimiter);
    }
    return builder.toString().getBytes();
}
 

private StreamlineEvent getStreamlineEventFromTuple(Tuple tuple) {
    final Object event = tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT);
    if (event instanceof StreamlineEvent) {
        return getStreamlineEventWithStream((StreamlineEvent) event, tuple);
    } else {
        LOG.debug("Invalid tuple received. Tuple disregarded and rules not evaluated.\n\tTuple [{}]." +
                          "\n\tStreamlineEvent [{}].", tuple, event);
    }
    return null;
}
 

@Override
public void reloadBolt(Tuple tuple) {
    String msg = null;
    try {
        logger.info("Begin to reload local cache!");
        PropertiesHolder.reload();
        GlobalCache.initialize(datasource);
        ThreadLocalCache.reload();
        Command.initialize();
        if (producer != null) {
            producer.close();
        }
        this.producer = createProducer(context.getThisTaskId());
        msg = "heartbeat bolt reload successful!";
        logger.info("Heartbeat bolt was reloaded at:{}", System.currentTimeMillis());
    } catch (Exception e) {
        msg = e.getMessage();
        logger.error("Reload heartbeat bolt error", e);
    } finally {
        if (tuple != null) {
            EmitData data = (EmitData) tuple.getValueByField(Constants.EmitFields.DATA);
            ControlMessage message = data.get(EmitData.MESSAGE);
            CtlMessageResult result = new CtlMessageResult("heartbeat-bolt", msg);
            result.setOriginalMessage(message);
            CtlMessageResultSender sender = new CtlMessageResultSender(message.getType(), zkconnect);
            sender.send("heartbeat-bolt", result, false, true);
        }
    }
}
 

public void execute(Tuple input) {
    if (TupleUtils.isTick(input)) {
        collector.ack(input);
        return;
    }
    try {
        Command cmd = (Command) input.getValueByField(Constants.EmitFields.COMMAND);
        BoltCommandHandler handler = handlerManager.getHandler(cmd);
        handler.handle(input);
        this.collector.ack(input);
    } catch (Exception e) {
        this.collector.ack(input);
        logger.error("Heartbeat error, ignore this error", e);
    }
}
 

@Override
public void writeData(String dbSchema, String table, DbusMessage dbusMessage, Tuple input) {
    EmitData data = (EmitData) input.getValueByField(Constants.EmitFields.DATA);
    List<String> topics = topicProvider.provideTopics(dbSchema, table);
    if (topics == null || topics.isEmpty()) {
        logger.error("Can't find a topic to write the message!");
        this.collector.ack(input);
        return;
    }

    String message = dbusMessage.toString();
    ProducerRecord<String, String> record = new ProducerRecord<>(topics.get(0), BoltCommandHandlerHelper.buildKey(dbusMessage), message);
    reporter.report(message.getBytes().length, dbusMessage.getPayload().size());
    Object offsetObj = data.get(EmitData.OFFSET);
    String offset = offsetObj != null ? offsetObj.toString() : "0";
    producer.send(record, (metadata, exception) -> {
        if (exception != null) {
            synchronized (this.collector) {
                this.collector.fail(input);
            }
            logger.error("Write data to kafka error, original data:[schema:{}, table:{}, offset:{}]!", dbSchema, table, offset, exception);
        } else {
            synchronized (this.collector) {
                this.collector.ack(input);
            }
            logger.debug("[appender-kafka-writer] kafka-message was sent, original-offset:{}, key:{}", offset, record.key());
        }
    });
}
 

@Override
public void reloadBolt(Tuple tuple) {
    String msg = null;
    try {
        PropertiesHolder.reload();
        ThreadLocalCache.reload();
        if (producer != null) {
            producer.close();
        }

        producer = createProducer();
        msg = "kafka write bolt reload successful!";
        logger.info("Kafka writer bolt was reloaded at:{}", System.currentTimeMillis());
    } catch (Exception e) {
        msg = e.getMessage();
        throw new RuntimeException(e);
    } finally {
        if (tuple != null) {
            EmitData data = (EmitData) tuple.getValueByField(Constants.EmitFields.DATA);
            ControlMessage message = data.get(EmitData.MESSAGE);
            CtlMessageResult result = new CtlMessageResult("kafka-write-bolt", msg);
            result.setOriginalMessage(message);
            CtlMessageResultSender sender = new CtlMessageResultSender(message.getType(), zkconnect);
            sender.send("kafka-write-bolt-" + context.getThisTaskId(), result, false, true);
        }
    }
}
 

private StreamlineEvent getStreamlineEventFromTuple(Tuple input) {
    Object event = null;
    if (input != null && input.contains(StreamlineEvent.STREAMLINE_EVENT)) {
         event = input.getValueByField(StreamlineEvent.STREAMLINE_EVENT);
        if(event instanceof StreamlineEvent) {
            return (StreamlineEvent) event;
        }
    }
    LOG.debug("Ignoring input tuple field [{}] because it does not contain object of type StreamlineEvent [{}]",
            StreamlineEvent.STREAMLINE_EVENT, event);
    return null;
}
 

@Override
public void execute(Tuple tuple) {
    String url = tuple.getStringByField("url");
    // Distinguish the value used for indexing
    // from the one used for the status
    String normalisedurl = valueForURL(tuple);
    Metadata metadata = (Metadata) tuple.getValueByField("metadata");
    String text = tuple.getStringByField("text");

    // tested by the TestOutputCollector
    boolean keep = filterDocument(metadata);
    if (!keep) {
        _collector.ack(tuple);
        return;
    }

    // display text of the document?
    if (fieldNameForText() != null) {
        fields.put(fieldNameForText(), trimText(text));
    }

    if (fieldNameForURL() != null) {
        fields.put(fieldNameForURL(), normalisedurl);
    }

    // which metadata to display?
    Map<String, String[]> keyVals = filterMetadata(metadata);

    Iterator<String> iterator = keyVals.keySet().iterator();
    while (iterator.hasNext()) {
        String fieldName = iterator.next();
        String[] values = keyVals.get(fieldName);
        for (String value : values) {
            fields.put(fieldName, value);
        }
    }

    _collector.ack(tuple);
}
 

@Override
public void execute(Tuple input, BasicOutputCollector basicOutputCollector) {
    Rankings rankings = (Rankings) input.getValueByField("rankings");
    /**
     * TODO:此处2个步骤的操作应该合并成一个lua操作，不过考虑到更新频率低，并且设置了globalGrouping，已经不存在并发状况了
     */
    hashOperations.getOperations().delete(WORD_COUNT_TOP_N_REAL_TIME_KEY);
    rankings.getRankings().forEach(rankable -> {
        String word = (String) rankable.getObject();
        long count = rankable.getCount();
        hashOperations.put(WORD_COUNT_TOP_N_REAL_TIME_KEY, word, count);
    });

}
 

@Override
public void handle(Tuple tuple) {
    EmitData data = (EmitData) tuple.getValueByField(Constants.EmitFields.DATA);
    List<PairWrapper<String, Object>> wrapperList = data.get(EmitData.MESSAGE);
    DataTable table = data.get(EmitData.DATA_TABLE);
    MetaVersion ver = data.get(EmitData.VERSION);
    for (PairWrapper<String, Object> wrapper : wrapperList) {
        Object packet = wrapper.getPairValue("PACKET");
        if (packet == null) {
            logger.warn("[appender-heartbeat] data error. wrapper:{}", JSON.toJSONString(wrapper));
            continue;
        }
        JSONObject json = JSON.parseObject(packet.toString());
        String dbSchema = wrapper.getPairValue("SCHEMA_NAME").toString();
        String tableName = wrapper.getPairValue("TABLE_NAME").toString();

        String pos = generateUmsId(wrapper.getProperties(Constants.MessageBodyKey.POS).toString());

        // oracle 数据格式：2017-03-24 14:28:00.995660，mysql格式：2017-03-24 14:28:00.995
        // 暂时不要统一
        //String opts = wrapper.getProperties(Constants.MessageBodyKey.OP_TS).toString().substring(0, 23);
        String opts = wrapper.getProperties(Constants.MessageBodyKey.OP_TS).toString();
        long time = json.getLong("time");
        long txTime = json.getLong("txTime");

        String targetTopic = listener.getTargetTopic(dbSchema, tableName);
        // 发送Heartbeat
        listener.sendHeartbeat(message(table, ver, pos, opts), targetTopic, buildKey(time + "|" + txTime, table, ver));
    }
}
 

@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    Fields fields = tuple.getFields();
    Assert.assertEquals(1, fields.size());
    TestRecord record = (TestRecord) tuple.getValueByField(OUTPUT_FIELD_NAME);
    record.assertIsValid();
    LOG.info("Test passed");
}
 

@Override
public void handle(Tuple tuple) {
    try {
        EmitData emitData = (EmitData) tuple.getValueByField(Constants.EmitFields.DATA);
        String entry = emitData.get(EmitData.MESSAGE);
        long offset = emitData.get(EmitData.OFFSET);
        logger.debug("[BEGIN] receive data,offset:{}", offset);

        //long pos = Long.parseLong(msgEntry.getEntryHeader().getPos()); //TODO 不知道Mongo的pos是多少
        syncMeta(entry, 0, offset, tuple);
    } catch (Exception e) {
        logger.error("Error when processing data", e);
        throw new RuntimeException(e);
    }
}
 

@Override
public void execute(Tuple tuple)
{
  if (tuple.getSourceStreamId().equals(StormConstants.ENCROWCALCBOLT_FLUSH_SIG))
  {
    numFlushSignals += 1;
    logger.debug("Received  {} flush signals out of {}", numFlushSignals, totalFlushSignals);

    // Need to receive notice from all EncRowCalcBolts in order to flush.
    if (numFlushSignals == totalFlushSignals)
    {
      logger.debug("Received signal to flush in EncColMultBolt. Outputting {} results.", resultsMap.keySet().size());
      for (Long key : resultsMap.keySet())
        // key = column Id, value = aggregated product
        outputCollector.emit(StormConstants.ENCCOLMULTBOLT_ID, new Values(key, resultsMap.get(key)));
      resultsMap.clear();

      // Send signal to OutputBolt to write output and notify EncRowCalcBolt that results have been flushed.
      outputCollector.emit(StormConstants.ENCCOLMULTBOLT_ID, new Values(-1L, BigInteger.ZERO));
      outputCollector.emit(StormConstants.ENCCOLMULTBOLT_SESSION_END, new Values(1));
      numFlushSignals = 0;
    }
  }
  else
  {
    // Data tuple received. Do column multiplication.

    long colIndex = tuple.getLongByField(StormConstants.COLUMN_INDEX_ERC_FIELD);
    BigInteger colVal1 = (BigInteger) tuple.getValueByField(StormConstants.ENCRYPTED_VALUE_FIELD);

    logger.debug("Received tuple in ECM, multiplying {} to col {}", colVal1, colIndex);

    if (resultsMap.containsKey(colIndex))
    {
      BigInteger colMult = colVal1.multiply(resultsMap.get(colIndex));
      resultsMap.put(colIndex, colMult.mod(nSquared));
    }
    else
    {
      resultsMap.put(colIndex, colVal1);
    }
  }
  outputCollector.ack(tuple);
}
 

@Override
public byte[] rowKey(Tuple tuple) {
    StreamlineEvent event = (StreamlineEvent) tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT);
    return toBytes((rowKeyField != null && !rowKeyField.isEmpty()) ? StreamlineRuntimeUtil.getFieldValue(event, rowKeyField) : event.getId());
}
 

/**
 * Returns the value to be used as the URL for indexing purposes, if present
 * the canonical value is used instead
 */
protected String valueForURL(Tuple tuple) {

    String url = tuple.getStringByField("url");
    Metadata metadata = (Metadata) tuple.getValueByField("metadata");

    // functionality deactivated
    if (StringUtils.isBlank(canonicalMetadataParamName)) {
        return url;
    }

    String canonicalValue = metadata.getFirstValue(canonicalMetadataName);

    // no value found?
    if (StringUtils.isBlank(canonicalValue)) {
        return url;
    }

    try {
        URL sURL = new URL(url);
        URL canonical = URLUtil.resolveURL(sURL, canonicalValue);

        String sDomain = PaidLevelDomain.getPLD(sURL.getHost());
        String canonicalDomain = PaidLevelDomain
                .getPLD(canonical.getHost());

        // check that the domain is the same
        if (sDomain.equalsIgnoreCase(canonicalDomain)) {
            return canonical.toExternalForm();
        } else {
            LOG.info(
                    "Canonical URL references a different domain, ignoring in {} ",
                    url);
        }
    } catch (MalformedURLException e) {
        LOG.error("Malformed canonical URL {} was found in {} ",
                canonicalValue, url);
    }

    return url;
}