org.apache.storm.tuple.Tuple#getStringByField

Source File: RedirectionBolt.java From storm-crawler with Apache License 2.0

6 votes

@Override
public void execute(Tuple tuple) {
    String url = tuple.getStringByField("url");
    byte[] content = tuple.getBinaryByField("content");
    Metadata metadata = (Metadata) tuple.getValueByField("metadata");
    String text = tuple.getStringByField("text");

    Values v = new Values(url, content, metadata, text);

    // if there is a text - no need to parse it again
    if (StringUtils.isNotBlank(text)) {
        collector.emit(tuple, v);
    } else {
        collector.emit("tika", tuple, v);
    }

    collector.ack(tuple);
}

Source File: TestBolt.java From springBoot-study with Apache License 2.0

6 votes

/**
	 * execute()方法是Bolt实现的核心。
	 * 也就是执行方法，每次Bolt从流接收一个订阅的tuple，都会调用这个方法。
	 */
	@Override
	public void execute(Tuple tuple) {
		/**
		 * 接受消息可以使用这两种方式进行接收。
		 * 个人推荐第二种。
		 */
//		String msg=tuple.getString(0);
		String msg=tuple.getStringByField("test");
		//这里我们就不做消息的处理，只打印
	    System.out.println("Bolt第"+count+"接受的消息:"+msg);	
	    count++;
	    /**
	     * 
         * 没次调用处理一个输入的tuple，所有的tuple都必须在一定时间内应答。
         * 可以是ack或者fail。否则，spout就会重发tuple。
         * 如果继承的是IRichBolt，则需要手动ack。
         * 这里就不用了,BaseRichBolt会自动帮我们应答。
	     */
//	    collector.ack(tuple);
	}

Source File: InsertBolt.java From springBoot-study with Apache License 2.0

6 votes

@Override
public void execute(Tuple tuple) {
	String msg=tuple.getStringByField(Constants.FIELD);
	try{
		List<User> listUser =JSON.parseArray(msg,User.class);
		//移除age小于10的数据
		if(listUser!=null&&listUser.size()>0){
			Iterator<User> iterator = listUser.iterator();
			 while (iterator.hasNext()) {
				 User user = iterator.next();
				 if (user.getAge()<10) {
					 logger.warn("Bolt移除的数据:{}",user);
					 iterator.remove();
				 }
			 }
			if(listUser!=null&&listUser.size()>0){
				userService.insertBatch(listUser);
			}
		}
	}catch(Exception e){
		logger.error("Bolt的数据处理失败!数据:{}",msg,e);
	}
}

Source File: HeartbeatHandler.java From DBus with Apache License 2.0

6 votes

@Override
public void handle(Tuple tuple) {
    EmitData data = (EmitData) tuple.getValueByField(Constants.EmitFields.DATA);
    List<PairWrapper<String, Object>> wrapperList = data.get(EmitData.MESSAGE);
    if (wrapperList != null && !wrapperList.isEmpty()) {
        for (PairWrapper<String, Object> wrapper : wrapperList) {
            HeartbeatPulse pulse = HeartbeatPulse.build(wrapper.pairs2map());
            if (logger.isDebugEnabled()) {
                Object offset = data.get(EmitData.OFFSET);
                HeartBeatPacket packet = HeartBeatPacket.parse(pulse.getPacket());
                SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
                String groupId = tuple.getStringByField(Constants.EmitFields.GROUP_FIELD);
                logger.debug("[heartbeat] {} offset:{} ts:{}, time:{}", groupId, offset == null ? -1 : offset, packet.getTxtime(), format.format(new Date(packet.getTxtime())));
            }
            reporter.mark(pulse);
        }
    }

    handler.handle(tuple);
    this.listener.getOutputCollector().ack(tuple);
}

Source File: TestBolt.java From java-study with Apache License 2.0

6 votes

/**
	 * execute()方法是Bolt实现的核心。
	 * 也就是执行方法，每次Bolt从流接收一个订阅的tuple，都会调用这个方法。
	 */
	@Override
	public void execute(Tuple tuple) {
		/**
		 * 接受消息可以使用这两种方式进行接收。
		 * 个人推荐第二种。
		 */
//		String msg=tuple.getString(0);
		String msg=tuple.getStringByField("test");
		//这里我们就不做消息的处理，只打印
	    System.out.println("Bolt第"+count+"接受的消息:"+msg);	
	    count++;
	    /**
	     * 
         * 没次调用处理一个输入的tuple，所有的tuple都必须在一定时间内应答。
         * 可以是ack或者fail。否则，spout就会重发tuple。
         * 如果继承的是IRichBolt，则需要手动ack。
         * 这里就不用了,BaseRichBolt会自动帮我们应答。
	     */
//	    collector.ack(tuple);
	}

Source File: LocalWordCountRedisStormTopology.java From 163-bigdate-note with GNU General Public License v3.0

6 votes

/**
 * 1.获取每个单词
 * 2.对所有单词进行汇总
 * 3.输出
 *
 * @param input
 */
@Override
public void execute(Tuple input) {
    //获取每个单词
    String word = input.getStringByField("word");

    //对所有单词汇总
    Integer count = map.get(word);
    if (count == null) {
        count = 0;
    }
    count++;

    map.put(word, count);

    //输出
    this.collector.emit(new Values(word, map.get(word)));
}

Source File: AdvertisingTopology.java From yahoo-streaming-benchmark with Apache License 2.0

5 votes

@Override
public void execute(Tuple tuple) {
    String ad_id = tuple.getStringByField("ad_id");
    String campaign_id = this.redisAdCampaignCache.execute(ad_id);
    if(campaign_id == null) {
        _collector.fail(tuple);
        return;
    }
    _collector.emit(tuple, new Values(campaign_id,
                                      tuple.getStringByField("ad_id"),
                                      tuple.getStringByField("event_time")));
    _collector.ack(tuple);
}

Source File: AdvertisingTopology.java From yahoo-streaming-benchmark with Apache License 2.0

5 votes

@Override
public void execute(Tuple tuple) {

    String campaign_id = tuple.getStringByField("campaign_id");
    String event_time = tuple.getStringByField("event_time");

   this.campaignProcessorCommon.execute(campaign_id, event_time);
    _collector.ack(tuple);
}

Source File: TestBolt.java From java-study with Apache License 2.0

5 votes

/**
 * execute()方法是Bolt实现的核心。
 * 也就是执行方法，每次Bolt从流接收一个订阅的tuple，都会调用这个方法。
 */
@Override
public void execute(Tuple tuple) {
	String msg=tuple.getStringByField("word");
    System.out.println("开始分割单词:"+msg);
       String[] words = msg.toLowerCase().split(" ");
       for (String word : words) {
           this.collector.emit(new Values(word));//向下一个bolt发射数据
       } 

}

Source File: AdvertisingTopology.java From streaming-benchmarks with Apache License 2.0

5 votes

@Override
public void execute(Tuple tuple) {

    String campaign_id = tuple.getStringByField("campaign_id");
    String event_time = tuple.getStringByField("event_time");

   this.campaignProcessorCommon.execute(campaign_id, event_time);
    _collector.ack(tuple);
}

Source File: SplitSentenceBolt.java From java-study with Apache License 2.0

5 votes

/**
 * SplitSentenceBolt核心功能是在类IBolt定义execute()方法，这个方法是IBolt接口中定义。
 * 每次Bolt从流接收一个订阅的tuple，都会调用这个方法。
 * 本例中,收到的元组中查找“sentence”的值,
 * 并将该值拆分成单个的词,然后按单词发出新的tuple。
 */
public void execute(Tuple input) {
    // TODO Auto-generated method stub
    String sentence = input.getStringByField("sentence");
    String[] words = sentence.split(" ");
    for (String word : words) {
        this.collector.emit(new Values(word));//向下一个bolt发射数据
    }       
}

Source File: ReportBolt.java From java-study with Apache License 2.0

5 votes

public void execute(Tuple input) {
    // TODO Auto-generated method stub

    String word = input.getStringByField("word");
    Long count = input.getLongByField("count");
    this.counts.put(word, count);

    //实时输出
    System.out.println("结果:"+this.counts);
}

Source File: LocalWordCountStormTopology.java From 163-bigdate-note with GNU General Public License v3.0

5 votes

/**
 * 对line按照逗号进行分割
 *
 * @param input
 */
@Override
public void execute(Tuple input) {
    String line = input.getStringByField("line");
    String[] words = line.split(" ");
    for (String word : words) {
        this.collector.emit(new Values(word));
    }
}

Source File: AdvertisingTopology.java From streaming-benchmarks with Apache License 2.0

5 votes

@Override
public void execute(Tuple tuple) {
    String ad_id = tuple.getStringByField("ad_id");
    String campaign_id = this.redisAdCampaignCache.execute(ad_id);
    if(campaign_id == null) {
        _collector.fail(tuple);
        return;
    }
    _collector.emit(tuple, new Values(campaign_id,
                                      tuple.getStringByField("ad_id"),
                                      tuple.getStringByField("event_time")));
    _collector.ack(tuple);
}

Source File: DummyIndexer.java From storm-crawler with Apache License 2.0

5 votes

@Override
public void execute(Tuple tuple) {
    String url = tuple.getStringByField("url");
    Metadata metadata = (Metadata) tuple.getValueByField("metadata");

    _collector.emit(
            com.digitalpebble.stormcrawler.Constants.StatusStreamName,
            tuple, new Values(url, metadata, Status.FETCHED));
    _collector.ack(tuple);
}

Source File: FeedDetectorBolt.java From news-crawl with Apache License 2.0

4 votes

@Override
public void execute(Tuple tuple) {
    Metadata metadata = (Metadata) tuple.getValueByField("metadata");

    byte[] content = tuple.getBinaryByField("content");
    String url = tuple.getStringByField("url");

    boolean isFeed = Boolean.valueOf(metadata.getFirstValue(isFeedKey));

    if (!isFeed) {
        String ct = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE);
        if (ct != null) {
            for (String clue : mimeTypeClues) {
                if (ct.contains(clue)) {
                    isFeed = true;
                    metadata.setValue(isFeedKey, "true");
                    LOG.info("Feed detected from content type <{}> for {}",
                            ct, url);
                    break;
                }
            }
        }
    }

    if (!isFeed) {
        if (contentDetector.matches(content)) {
            isFeed = true;
            metadata.setValue(isFeedKey, "true");
            LOG.info("Feed detected from content: {}", url);
        }
    }

    if (isFeed) {
        // do not parse but run parse filters
        ParseResult parse = new ParseResult();
        ParseData parseData = parse.get(url);
        parseData.setMetadata(metadata);
        parseFilters.filter(url, content, null, parse);
        // emit status
        collector.emit(Constants.StatusStreamName, tuple,
                new Values(url, metadata, Status.FETCHED));
    } else {
        // pass on
        collector.emit(tuple, tuple.getValues());
    }
    collector.ack(tuple);
}

Source File: NewsSiteMapDetectorBolt.java From news-crawl with Apache License 2.0

4 votes

@Override
public void execute(Tuple tuple) {
    Metadata metadata = (Metadata) tuple.getValueByField("metadata");

    byte[] content = tuple.getBinaryByField("content");
    String url = tuple.getStringByField("url");

    boolean isSitemap = Boolean.valueOf(
            metadata.getFirstValue(SiteMapParserBolt.isSitemapKey));
    boolean isNewsSitemap = Boolean.valueOf(
            metadata.getFirstValue(NewsSiteMapParserBolt.isSitemapNewsKey));

    if (!isNewsSitemap || !isSitemap) {
        int match = contentDetector.getFirstMatch(content);
        if (match >= 0) {
            // a sitemap, not necessarily a news sitemap
            isSitemap = true;
            metadata.setValue(SiteMapParserBolt.isSitemapKey, "true");
            if (match <= NewsSiteMapParserBolt.contentCluesSitemapNewsMatchUpTo) {
                isNewsSitemap = true;
                LOG.info("{} detected as news sitemap based on content",
                        url);
                metadata.setValue(NewsSiteMapParserBolt.isSitemapNewsKey,
                        "true");
            }
        }
    }

    if (isSitemap) {
        // do not parse but run parse filters
        ParseResult parse = new ParseResult();
        ParseData parseData = parse.get(url);
        parseData.setMetadata(metadata);
        parseFilters.filter(url, content, null, parse);
        // emit status
        collector.emit(Constants.StatusStreamName, tuple,
                new Values(url, metadata, Status.FETCHED));
    } else {
        // pass on
        collector.emit(tuple, tuple.getValues());
    }
    collector.ack(tuple);
}

Source File: IndexerBolt.java From storm-crawler with Apache License 2.0

4 votes

@Override
public void execute(Tuple tuple) {

    String url = tuple.getStringByField("url");

    Metadata metadata = (Metadata) tuple.getValueByField("metadata");
    String text = tuple.getStringByField("text");

    boolean keep = filterDocument(metadata);
    if (!keep) {
        eventCounter.scope("Filtered").incrBy(1);
        // treat it as successfully processed even if
        // we do not index it
        _collector.emit(StatusStreamName, tuple, new Values(url, metadata,
                Status.FETCHED));
        _collector.ack(tuple);
        return;
    }

    try {
        SolrInputDocument doc = new SolrInputDocument();

        // index text content
        if (fieldNameForText() != null) {
            doc.addField(fieldNameForText(), trimText(text));
        }

        // url
        if (fieldNameForURL() != null) {
            // Distinguish the value used for indexing
            // from the one used for the status
            String normalisedurl = valueForURL(tuple);
            doc.addField(fieldNameForURL(), normalisedurl);
        }

        // select which metadata to index
        Map<String, String[]> keyVals = filterMetadata(metadata);

        Iterator<String> iterator = keyVals.keySet().iterator();
        while (iterator.hasNext()) {
            String fieldName = iterator.next();
            String[] values = keyVals.get(fieldName);
            for (String value : values) {
                doc.addField(fieldName, value);
            }
        }

        connection.getClient().add(doc);

        eventCounter.scope("Indexed").incrBy(1);

        _collector.emit(StatusStreamName, tuple, new Values(url, metadata,
                Status.FETCHED));
        _collector.ack(tuple);

    } catch (Exception e) {
        LOG.error("Send update request to SOLR failed due to {}", e);
        _collector.fail(tuple);
    }
}

Source File: StatusUpdaterBolt.java From storm-crawler with Apache License 2.0

4 votes

@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    LOG.debug("afterBulk [{}] with {} responses", executionId,
            request.numberOfActions());
    long msec = response.getTook().getMillis();
    eventCounter.scope("bulks_received").incrBy(1);
    eventCounter.scope("bulk_msec").incrBy(msec);
    Iterator<BulkItemResponse> bulkitemiterator = response.iterator();
    int itemcount = 0;
    int acked = 0;
    int failurecount = 0;

    synchronized (waitAck) {
        while (bulkitemiterator.hasNext()) {
            BulkItemResponse bir = bulkitemiterator.next();
            itemcount++;
            String id = bir.getId();
            BulkItemResponse.Failure f = bir.getFailure();
            boolean failed = false;
            if (f != null) {
                // already discovered
                if (f.getStatus().equals(RestStatus.CONFLICT)) {
                    eventCounter.scope("doc_conflicts").incrBy(1);
                    LOG.debug("Doc conflict ID {}", id);
                } else {
                    LOG.error("Update ID {}, failure: {}", id, f);
                    failed = true;
                }
            }
            List<Tuple> xx = waitAck.getIfPresent(id);
            if (xx != null) {
                LOG.debug("Acked {} tuple(s) for ID {}", xx.size(), id);
                for (Tuple x : xx) {
                    if (!failed) {
                        String url = x.getStringByField("url");
                        acked++;
                        // ack and put in cache
                        LOG.debug("Acked {} with ID {}", url, id);
                        super.ack(x, url);
                    } else {
                        failurecount++;
                        _collector.fail(x);
                    }
                }
                waitAck.invalidate(id);
            } else {
                LOG.warn("Could not find unacked tuple for {}", id);
            }
        }

        LOG.info(
                "Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
                executionId, itemcount, waitAck.size(), acked, failurecount);
        if (waitAck.size() > 0 && LOG.isDebugEnabled()) {
            for (String kinaw : waitAck.asMap().keySet()) {
                LOG.debug(
                        "Still in wait ack after bulk response [{}] => {}",
                        executionId, kinaw);
            }
        }
    }
}

Source File: IndexerBolt.java From storm-crawler with Apache License 2.0

4 votes

@Override
public void execute(Tuple tuple) {

    String url = tuple.getStringByField("url");

    // Distinguish the value used for indexing
    // from the one used for the status
    String normalisedurl = valueForURL(tuple);

    LOG.info("Indexing {} as {}", url, normalisedurl);

    Metadata metadata = (Metadata) tuple.getValueByField("metadata");
    String text = tuple.getStringByField("text");

    boolean keep = filterDocument(metadata);
    if (!keep) {
        LOG.info("Filtered {}", url);
        eventCounter.scope("Filtered").incrBy(1);
        // treat it as successfully processed even if
        // we do not index it
        _collector.emit(StatusStreamName, tuple, new Values(url, metadata,
                Status.FETCHED));
        _collector.ack(tuple);
        return;
    }

    String docID = org.apache.commons.codec.digest.DigestUtils
            .sha256Hex(normalisedurl);

    try {
        XContentBuilder builder = jsonBuilder().startObject();

        // display text of the document?
        if (fieldNameForText() != null) {
            builder.field(fieldNameForText(), trimText(text));
        }

        // send URL as field?
        if (fieldNameForURL() != null) {
            builder.field(fieldNameForURL(), normalisedurl);
        }

        // which metadata to display?
        Map<String, String[]> keyVals = filterMetadata(metadata);

        Iterator<String> iterator = keyVals.keySet().iterator();
        while (iterator.hasNext()) {
            String fieldName = iterator.next();
            String[] values = keyVals.get(fieldName);
            if (values.length == 1) {
                builder.field(fieldName, values[0]);
            } else if (values.length > 1) {
                builder.array(fieldName, values);
            }
        }

        builder.endObject();

        String sha256hex = org.apache.commons.codec.digest.DigestUtils
                .sha256Hex(normalisedurl);

        IndexRequest indexRequest = new IndexRequest(getIndexName(metadata))
                .source(builder).id(sha256hex);

        DocWriteRequest.OpType optype = DocWriteRequest.OpType.INDEX;

        if (create) {
            optype = DocWriteRequest.OpType.CREATE;
        }

        indexRequest.opType(optype);

        if (pipeline != null) {
            indexRequest.setPipeline(pipeline);
        }

        connection.getProcessor().add(indexRequest);

        eventCounter.scope("Indexed").incrBy(1);
        perSecMetrics.scope("Indexed").update(1);

        synchronized (waitAck) {
            waitAck.put(docID, tuple);
        }
    } catch (IOException e) {
        LOG.error("Error building document for ES", e);
        // do not send to status stream so that it gets replayed
        _collector.fail(tuple);
        if (docID != null) {
            synchronized (waitAck) {
                waitAck.invalidate(docID);
            }
        }
    }
}

Java Code Examples for org.apache.storm.tuple.Tuple#getStringByField()