storm.trident.operation.TridentCollector Java Examples

private void emitCrossJoin(JoinState state, TridentCollector collector, int overrideIndex, TridentTuple overrideTuple) {
    List<List>[] sides = state.sides;
    int[] indices = state.indices;
    for (int i = 0; i < indices.length; i++) {
        indices[i] = 0;
    }

    boolean keepGoing = true;
    // emit cross-join of all emitted tuples
    while (keepGoing) {
        List[] combined = new List[sides.length + 1];
        combined[0] = state.group;
        for (int i = 0; i < sides.length; i++) {
            if (i == overrideIndex) {
                combined[i + 1] = overrideTuple;
            } else {
                combined[i + 1] = sides[i].get(indices[i]);
            }
        }
        collector.emit(_factory.create(combined));
        keepGoing = increment(sides, indices, indices.length - 1, overrideIndex);
    }
}
 

public void complete(ChainedResult val, TridentCollector collector) {
    val.setFollowThroughCollector(collector);
    for (int i = 0; i < _aggs.length; i++) {
        _aggs[i].complete(val.objs[i], val.collectors[i]);
    }
    if (_aggs.length > 1) { // otherwise, tuples were emitted directly
        int[] indices = new int[val.collectors.length];
        for (int i = 0; i < indices.length; i++) {
            indices[i] = 0;
        }
        boolean keepGoing = true;
        // emit cross-join of all emitted tuples
        while (keepGoing) {
            List[] combined = new List[_aggs.length];
            for (int i = 0; i < _aggs.length; i++) {
                CaptureCollector capturer = (CaptureCollector) val.collectors[i];
                combined[i] = capturer.captured.get(indices[i]);
            }
            collector.emit(_fact.create(combined));
            keepGoing = increment(val.collectors, indices, indices.length - 1);
        }
    }
}
 

@Override
public void emitBatch(long batchId, TridentCollector collector) {
    List<List<Object>> batch = this.batches.get(batchId);
    if(batch == null){
        batch = new ArrayList<List<Object>>();
        if(index>=outputs.length && cycle) {
            index = 0;
        }
        for(int i=0; index < outputs.length && i < maxBatchSize; index++, i++) {
            batch.add(outputs[index]);
        }
        this.batches.put(batchId, batch);
    }
    for(List<Object> list : batch){
        collector.emit(list);
    }
}
 

@Override
public void aggregate(State state, TridentTuple tuple, TridentCollector collector) {
    T value1 = valueFromTuple(state.previousTuple);
    T value2 = valueFromTuple(tuple);

    log.debug("Aggregated tuple value in state [{}], and received tuple value [{}] in operation [{}]", value1, value2, this);

    if(value2 == null) {
        return;
    }

    if(value1 == null || compare(value1, value2) == value2) {
        state.previousTuple = tuple;
    }

}
 

@Override
public void emitBatch(long batchId, TridentCollector collector) {
    if(batchId > limit)                     //batchId starts from 1, batchId > limit means it has emitted ${limit} batches
    {
        try {
            Thread.sleep(1000);             //repeat sleep 1s until cluster is closed
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return;
    }

    List<List<Object>> batch;
    if(batches.containsKey(batchId)) {
        batch = batches.get(batchId);
    }
    else {
        batch = getBatchContent(maxBatchSize);
        this.batches.put(batchId, batch);
    }

    for(List<Object> tuple : batch) {
        collector.emit(tuple);
    }
}
 

private Map emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) {
    try {
        return failFastEmitNewPartitionBatch(attempt, collector, partition, lastMeta);
    } catch (FailedFetchException e) {
        LOG.warn("Failed to fetch from partition " + partition);
        if (lastMeta == null) {
            return null;
        } else {
            Map ret = new HashMap();
            ret.put("offset", lastMeta.get("nextOffset"));
            ret.put("nextOffset", lastMeta.get("nextOffset"));
            ret.put("partition", partition.partition);
            ret.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port));
            ret.put("topic", _config.topic);
            ret.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId));
            return ret;
        }
    }
}
 

@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta, TridentCollector collector) {
    File file = new File("click_thru_data.txt");
    try {
        BufferedReader br = new BufferedReader(new FileReader(file));
        String line = null;
        while ((line = br.readLine()) != null) {
            String[] data = line.split(" ");
            List<Object> tuple = new ArrayList<Object>();
            tuple.add(data[0]); // username
            tuple.add(data[1]); // campaign
            tuple.add(data[2]); // product
            tuple.add(data[3]); // click
            collector.emit(tuple);
        }
        br.close();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    String tweet = (String) tuple.getValue(0);
    LOG.error("SPLITTING TWEET [" + tweet + "]");
    Pattern p = Pattern.compile("[a-zA-Z]+");
    Matcher m = p.matcher(tweet);
    List<String> result = new ArrayList<String>();
    while (m.find()) {
        String word = m.group();
        if (word.length() > 0) {
            List<Object> newTuple = new ArrayList<Object>();
            newTuple.add(word);
            collector.emit(newTuple);
        }
    }
}
 

@Override
public void aggregate(Object[] arr, TridentTuple tuple, TridentCollector collector) {
    GroupCollector groupColl = (GroupCollector) arr[0];
    Map<List, Object> val = (Map) arr[1];
    TridentTuple group = _groupFactory.create((TridentTupleView) tuple);
    TridentTuple input = _inputFactory.create((TridentTupleView) tuple);
    Object curr;
    if (!val.containsKey(group)) {
        curr = _agg.init(arr[2], groupColl);
        val.put((List) group, curr);
    } else {
        curr = val.get(group);
    }
    groupColl.currGroup = group;
    _agg.aggregate(curr, input, groupColl);

}
 

@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta, TridentCollector collector) {
    int size = 0;
    LOG.debug("Getting batch for [" + tx.getTransactionId() + "]");
    while (size <= MAX_BATCH_SIZE) {
        List<Object> values = new ArrayList<Object>();
        try {
            LOG.debug("Waiting on work from [" + this.queueName + "]:[" + getQueue().size() + "]");
            values.add(getQueue().take());
            LOG.debug("Got work from [" + this.queueName + "]:[" + getQueue().size() + "]");
        } catch (InterruptedException ex) {
            // do something smart
        }
        collector.emit(values);
        size++;
    }
    LOG.info("Emitted [" + size + "] elements in [" + tx.getTransactionId() + "], [" + getQueue().size() + "] remain in queue.");
}
 

@Override
public void updateState(Snapshottable state, List<TridentTuple> tuples, TridentCollector collector) {
    if (tuples.size() != 1) {
        throw new IllegalArgumentException("Combiner state updater should receive a single tuple. Received: " + tuples.toString());
    }
    Object newVal = state.update(new CombinerValueUpdater(_agg, tuples.get(0).getValue(0)));
    collector.emit(new Values(newVal));
}
 

@Override
public void aggregate(SingleEmitState val, TridentTuple tuple, TridentCollector collector) {
    if (!val.received) {
        val.state = _agg.init(val.batchId, collector);
        val.received = true;
    }
    _agg.aggregate(val.state, tuple, collector);
}
 

@Override
public void emitBatch(long batchId, TridentCollector collector) {
    // emit batchSize fake tweets
    for (int i = 0; i < batchSize; i++) {
        collector.emit(fakeTweetGenerator.getNextTweet());
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    if(splitter==null){
        splitter = Splitter.on(on);
    }

    String string = tuple.getString(0);
    for (String spilt : splitter.split(string)) {
        collector.emit(new Values(spilt));
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    if(extracter == null) extracter = new ContentExtracter();

    String rawTweetJson = (String)tuple.get(0);
    Status parsed = parse(rawTweetJson);
    User user = parsed.getUser();

    for (Content content : extracter.extract(parsed)) {
        collector.emit(new Values(parsed, content, user));
    }
}
 

public void setFollowThroughCollector(TridentCollector collector) {
    if (collectors.length > 1) {
        for (TridentCollector c : collectors) {
            ((CaptureCollector) c).setCollector(collector);
        }
    }
}
 

@Override
public void emitBatch(final TransactionAttempt tx, final Integer coordinatorMeta,
        final TridentCollector collector) {
    if(_savedCoordinatorMeta == null || !_savedCoordinatorMeta.equals(coordinatorMeta)) {
        List<ISpoutPartition> partitions = _emitter.getOrderedPartitions(coordinatorMeta);
        _partitionStates.clear();
        List<ISpoutPartition> myPartitions = new ArrayList<>();
        for(int i=_index; i < partitions.size(); i+=_numTasks) {
            ISpoutPartition p = partitions.get(i);
            String id = p.getId();
            myPartitions.add(p);
            _partitionStates.put(id, new EmitterPartitionState(new RotatingTransactionalState(_state, id), p));
        }
        _emitter.refreshPartitions(myPartitions);
        _savedCoordinatorMeta = coordinatorMeta;
    }
    for(EmitterPartitionState s: _partitionStates.values()) {
        RotatingTransactionalState state = s.rotatingState;
        final ISpoutPartition partition = s.partition;
        Object meta = state.getStateOrCreate(tx.getTransactionId(),
                new RotatingTransactionalState.StateInitializer() {
            @Override
            public Object init(long txid, Object lastState) {
                return _emitter.emitPartitionBatchNew(tx, collector, partition, lastState);
            }
        });
        // it's null if one of:
        // a) a later transaction batch was emitted before this, so we should skip this batch
        // b) if didn't exist and was created (in which case the StateInitializer was invoked and
        // it was emitted
        if (meta != null) {
            _emitter.emitPartitionBatch(tx, collector, partition, meta);
        }
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
	Matcher m = threeDigitAbbr.matcher(tuple.getString(0));
	if (m.matches()) {
		matches++;
	}

	if(start == 0) {
		start = System.currentTimeMillis();
	}
	received++;
	if(received % logfreq == 0) {
		long now = System.currentTimeMillis();


		// throughput for the last "logfreq" elements
		if(lastLog == -1) {
			// init (the first)
			lastLog = now;
			lastElements = received;
		} else {
			long timeDiff = now - lastLog;
			long elementDiff = received - lastElements;
			double ex = (1000/(double)timeDiff);
			LOG.info("During the last {} ms, we received {} elements. That's {} elements/second/core", timeDiff, elementDiff, elementDiff*ex);
			// reinit
			lastLog = now;
			lastElements = received;
		}
	}

	if(tuple.getLong(1) != 0) {
		long lat = System.currentTimeMillis() - tuple.getLong(1);
		LOG.info("Latency {} ms from same machine", lat);
	}
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
  List l = (List) tuple.getValue(0);
  if (l != null) {
    for (Object o : l) {
      collector.emit(new Values(o));
    }
  }
}
 

public void updateState(List<TridentTuple> tuples, TridentCollector tridentCollector) {
    try {
        this.options.execute(tuples);
    } catch (IOException e) {
        LOG.warn("Failing batch due to IOException.", e);
        throw new FailedException(e);
    }
}
 

@Override
public PriorityQueue init(Object batchId, TridentCollector collector) {
    return new PriorityQueue(_n, new Comparator<TridentTuple>() {
        @Override
        public int compare(TridentTuple t1, TridentTuple t2) {
            Comparable c1 = (Comparable) t1.getValueByField(_sortField);
            Comparable c2 = (Comparable) t2.getValueByField(_sortField);
            int ret = c1.compareTo(c2);
            if(_reverse) ret *= -1;
            return ret;
        }                
    });
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    if(tuple.get(0) == null){
        collector.emit(new Values(t));
    }else{
        collector.emit(new Values(tuple.get(0)));
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    String sentence = tuple.getString(0);
    for (String word : sentence.split(" ")) {
        collector.emit(new Values(word));
    }
}
 

@Override
public void complete(State val, TridentCollector collector) {
	Values values = new Values();
	values.add(val.count);
	for (String field : keepFields)
		values.add(val.fields.get(field));
	
	collector.emit(values);
}
 

@Test (dataProvider = "getIterable")
public void testExpand(Iterable iterable, int size)  {
  final TridentTuple tuple = mock(TridentTuple.class);
  final TridentCollector collector = mock(TridentCollector.class);
  when(tuple.getValue(0)).thenReturn(iterable);
  Expand expand = new Expand();
  expand.execute(tuple, collector);

  verify(tuple, times(1)).getValue(0);
  verify(collector, times(size)).emit(any(Values.class));
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    String sentence = tuple.getString(0);
    for (String word : sentence.split(" ")) {
        collector.emit(new Values(word));
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    String sentence = tuple.getString(0);
    for (String word : sentence.split(" ")) {
        collector.emit(new Values(word));
    }
}
 

@SuppressWarnings("rawtypes")
@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta, TridentCollector collector) {
    InputStream inputStream = null;
    File file = new File("fix_data.txt");
    try {
        inputStream = new BufferedInputStream(new FileInputStream(file));
        SimpleFixParser parser = new SimpleFixParser(inputStream);
        SimpleFixMessage msg = null;
        do {
            msg = parser.readFixMessage();
            if (null != msg) {
                FixMessageDto dto = new FixMessageDto();
                for (TagValue tagValue : msg.fields()) {
                    if (tagValue.tag().equals("6")) { // AvgPx
                        // dto.price = Double.valueOf((String) tagValue.value());
                        dto.price = new Double((int) (Math.random() * 100));
                    } else if (tagValue.tag().equals("35")) {
                        dto.msgType = (String) tagValue.value();
                    } else if (tagValue.tag().equals("55")) {
                        dto.symbol = (String) tagValue.value();
                    } else if (tagValue.tag().equals("11")) {
                        // dto.uid = (String) tagValue.value();
                        dto.uid = Integer.toString(uids.incrementAndGet());
                    }
                }
                new ObjectOutputStream(new ByteArrayOutputStream()).writeObject(dto);
                List<Object> message = new ArrayList<Object>();
                message.add(dto);
                collector.emit(message);
            }
        } while (msg != null);
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        IoUtils.closeSilently(inputStream);
    }
}
 

@Override
public void updateState(DruidState state, List<TridentTuple> tuples, TridentCollector collector) {
    //LOG.info("Updating [" + state + "]");
    for (TridentTuple tuple : tuples) {
        FixMessageDto message = (FixMessageDto) tuple.getValue(0);
        state.aggregateMessage(message);
    }
}
 

@Override
public void complete(Object[] arr, TridentCollector collector) {
    Map<List, Object> val = (Map) arr[1];
    GroupCollector groupColl = (GroupCollector) arr[0];
    for (Entry<List, Object> e : val.entrySet()) {
        groupColl.currGroup = e.getKey();
        _agg.complete(e.getValue(), groupColl);
    }
}