storm.trident.topology.TransactionAttempt Java Examples

@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta, TridentCollector collector) {
    int size = 0;
    LOG.debug("Getting batch for [" + tx.getTransactionId() + "]");
    while (size <= MAX_BATCH_SIZE) {
        List<Object> values = new ArrayList<Object>();
        try {
            LOG.debug("Waiting on work from [" + this.queueName + "]:[" + getQueue().size() + "]");
            values.add(getQueue().take());
            LOG.debug("Got work from [" + this.queueName + "]:[" + getQueue().size() + "]");
        } catch (InterruptedException ex) {
            // do something smart
        }
        collector.emit(values);
        size++;
    }
    LOG.info("Emitted [" + size + "] elements in [" + tx.getTransactionId() + "], [" + getQueue().size() + "] remain in queue.");
}
 

@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta, TridentCollector collector) {
    File file = new File("click_thru_data.txt");
    try {
        BufferedReader br = new BufferedReader(new FileReader(file));
        String line = null;
        while ((line = br.readLine()) != null) {
            String[] data = line.split(" ");
            List<Object> tuple = new ArrayList<Object>();
            tuple.add(data[0]); // username
            tuple.add(data[1]); // campaign
            tuple.add(data[2]); // product
            tuple.add(data[3]); // click
            collector.emit(tuple);
        }
        br.close();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * re-emit the batch described by the meta data provided
 *
 * @param attempt
 * @param collector
 * @param partition
 * @param meta
 */
private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) {
    LOG.info("re-emitting batch, attempt " + attempt);
    String instanceId = (String) meta.get("instanceId");
    if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
        SimpleConsumer consumer = _connections.register(partition);
        long offset = (Long) meta.get("offset");
        long nextOffset = (Long) meta.get("nextOffset");
        ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset);
        for (MessageAndOffset msg : msgs) {
            if (offset == nextOffset) {
                break;
            }
            if (offset > nextOffset) {
                throw new RuntimeException("Error when re-emitting batch. overshot the end offset");
            }
            emit(collector, msg.message());
            offset = msg.nextOffset();
        }
    }
}
 

private Map emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) {
    try {
        return failFastEmitNewPartitionBatch(attempt, collector, partition, lastMeta);
    } catch (FailedFetchException e) {
        LOG.warn("Failed to fetch from partition " + partition);
        if (lastMeta == null) {
            return null;
        } else {
            Map ret = new HashMap();
            ret.put("offset", lastMeta.get("nextOffset"));
            ret.put("nextOffset", lastMeta.get("nextOffset"));
            ret.put("partition", partition.partition);
            ret.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port));
            ret.put("topic", _config.topic);
            ret.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId));
            return ret;
        }
    }
}
 

@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    TransactionAttempt attempt = (TransactionAttempt) tuple.getValue(0);

    if (tuple.getSourceStreamId().equals(MasterBatchCoordinator.SUCCESS_STREAM_ID)) {
        _state.cleanupBefore(attempt.getTransactionId());
        _coord.success(attempt.getTransactionId());
    } else {
        long txid = attempt.getTransactionId();
        Object prevMeta = _state.getPreviousState(txid);
        Object meta = _coord.initializeTransaction(txid, prevMeta, _state.getState(txid));
        _state.overrideState(txid, meta);
        collector.emit(MasterBatchCoordinator.BATCH_STREAM_ID, new Values(attempt, meta));
    }

}
 

@Override
public void execute(BatchInfo info, Tuple input) {
    // there won't be a BatchInfo for the success stream
    TransactionAttempt attempt = (TransactionAttempt) input.getValue(0);
    if (input.getSourceStreamId().equals(MasterBatchCoordinator.COMMIT_STREAM_ID)) {
        if (attempt.equals(_activeBatches.get(attempt.getTransactionId()))) {
            ((ICommitterTridentSpout.Emitter) _emitter).commit(attempt);
            _activeBatches.remove(attempt.getTransactionId());
        } else {
            throw new FailedException("Received commit for different transaction attempt");
        }
    } else if (input.getSourceStreamId().equals(MasterBatchCoordinator.SUCCESS_STREAM_ID)) {
        // valid to delete before what's been committed since
        // those batches will never be accessed again
        _activeBatches.headMap(attempt.getTransactionId()).clear();
        _emitter.success(attempt);
    } else {
        _collector.setBatch(info.batchId);
        _emitter.emitBatch(attempt, input.getValue(1), _collector);
        _activeBatches.put(attempt.getTransactionId(), attempt);
    }
}
 

@Override
public void finishBatch(ProcessorContext processorContext) {
    _collector.setContext(processorContext);
    Object batchId = processorContext.batchId;
    // since this processor type is a committer, this occurs in the commit phase
    List<TridentTuple> buffer = (List) processorContext.state[_context.getStateIndex()];

    // don't update unless there are tuples
    // this helps out with things like global partition persist, where multiple tasks may still
    // exist for this processor. Only want the global one to do anything
    // this is also a helpful optimization that state implementations don't need to manually do
    if (buffer.size() > 0) {
        Long txid = null;
        // this is to support things like persisting off of drpc stream, which is inherently unreliable
        // and won't have a tx attempt
        if (batchId instanceof TransactionAttempt) {
            txid = ((TransactionAttempt) batchId).getTransactionId();
        }
        _state.beginCommit(txid);
        _updater.updateState(_state, buffer, _collector);
        _state.commit(txid);
    }
}
 

public IOpaquePartitionedTridentSpout.Emitter<GlobalPartitionInformation, Partition, Map> asOpaqueEmitter() {

        return new IOpaquePartitionedTridentSpout.Emitter<GlobalPartitionInformation, Partition, Map>() {

            /**
             * Emit a batch of tuples for a partition/transaction.
             *
             * Return the metadata describing this batch that will be used as lastPartitionMeta
             * for defining the parameters of the next batch.
             */
            @Override
            public Map emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) {
                return emitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map);
            }

            @Override
            public void refreshPartitions(List<Partition> partitions) {
                refresh(partitions);
            }

            @Override
            public List<Partition> getOrderedPartitions(GlobalPartitionInformation partitionInformation) {
                return orderPartitions(partitionInformation);
            }

            @Override
            public void close() {
                clear();
            }
        };
    }
 

@Override
public void emitBatch(TransactionAttempt tx, Map<Integer, List<List<Object>>> coordinatorMeta, TridentCollector collector) {
    List<List<Object>> tuples = coordinatorMeta.get(_index);
    if (tuples != null) {
        for (List<Object> t : tuples) {
            collector.emit(t);
        }
    }
}
 

@Override
public void commit(TransactionAttempt attempt) {
    // this code here handles a case where a previous commit failed, and the partitions
    // changed since the last commit. This clears out any state for the removed partitions
    // for this txid.
    // we make sure only a single task ever does this. we're also guaranteed that
    // it's impossible for there to be another writer to the directory for that partition
    // because only a single commit can be happening at once. this is because in order for 
    // another attempt of the batch to commit, the batch phase must have succeeded in between.
    // hence, all tasks for the prior commit must have finished committing (whether successfully or not)
    if(_changedMeta && _index==0) {
        Set<String> validIds = new HashSet<>();
        for(ISpoutPartition p: _emitter.getOrderedPartitions(_savedCoordinatorMeta)) {
            validIds.add(p.getId());
        }
        for(String existingPartition: _state.list("")) {
            if(!validIds.contains(existingPartition)) {
                RotatingTransactionalState s = new RotatingTransactionalState(_state, existingPartition);
                s.removeState(attempt.getTransactionId());
            }
        }
        _changedMeta = false;
    }            
    
    Long txid = attempt.getTransactionId();
    Map<String, Object> metas = _cachedMetas.remove(txid);
    for(Entry<String, Object> entry: metas.entrySet()) {
        _partitionStates.get(entry.getKey()).rotatingState.overrideState(txid, entry.getValue());
    }
}
 

@Override
public void emitBatch(final TransactionAttempt tx, final Integer coordinatorMeta,
        final TridentCollector collector) {
    if(_savedCoordinatorMeta == null || !_savedCoordinatorMeta.equals(coordinatorMeta)) {
        List<ISpoutPartition> partitions = _emitter.getOrderedPartitions(coordinatorMeta);
        _partitionStates.clear();
        List<ISpoutPartition> myPartitions = new ArrayList<>();
        for(int i=_index; i < partitions.size(); i+=_numTasks) {
            ISpoutPartition p = partitions.get(i);
            String id = p.getId();
            myPartitions.add(p);
            _partitionStates.put(id, new EmitterPartitionState(new RotatingTransactionalState(_state, id), p));
        }
        _emitter.refreshPartitions(myPartitions);
        _savedCoordinatorMeta = coordinatorMeta;
    }
    for(EmitterPartitionState s: _partitionStates.values()) {
        RotatingTransactionalState state = s.rotatingState;
        final ISpoutPartition partition = s.partition;
        Object meta = state.getStateOrCreate(tx.getTransactionId(),
                new RotatingTransactionalState.StateInitializer() {
            @Override
            public Object init(long txid, Object lastState) {
                return _emitter.emitPartitionBatchNew(tx, collector, partition, lastState);
            }
        });
        // it's null if one of:
        // a) a later transaction batch was emitted before this, so we should skip this batch
        // b) if didn't exist and was created (in which case the StateInitializer was invoked and
        // it was emitted
        if (meta != null) {
            _emitter.emitPartitionBatch(tx, collector, partition, meta);
        }
    }
}
 

@Override
public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) {
    long txid = tx.getTransactionId();

    long now = System.currentTimeMillis();
    if (now - lastRotate > rotateTime) {
        Map<Long, List<Object>> failed = idsMap.rotate();
        for (Long id : failed.keySet()) {
            // TODO: this isn't right... it's not in the map anymore
            fail(id);
        }
        lastRotate = now;
    }

    if (idsMap.containsKey(txid)) {
        fail(txid);
    }

    _collector.reset(collector);
    if (!prepared) {
        _spout.open(_conf, _context, new SpoutOutputCollector(_collector));
        prepared = true;
    }
    for (int i = 0; i < _maxBatchSize; i++) {
        _spout.nextTuple();
        if(_collector.numEmitted < i) {
            break;
        }
    }
    idsMap.put(txid, _collector.ids);
    _collector.pendingCount = idsMap.size();

}
 

@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta,
                      TridentCollector collector) {
    for (int i = 0; i < 100; i++) {
        List<Object> sale = new ArrayList<Object>();
        sale.add(states[generator.nextInt(4)]);
        sale.add(products[generator.nextInt(4)]);
        sale.add(generator.nextInt(100));
        collector.emit(sale);
    }
}
 

@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta, TridentCollector collector) {
    for (Status status : result.getTweets()) {
        List<Object> tweets = new ArrayList<Object>();
        tweets.add(SEARCH_PHRASE);
        tweets.add(status.getText());
        collector.emit(tweets);
        System.out.println("Emitted [" + status.getText() + "]");
    }
}
 

@SuppressWarnings("rawtypes")
@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta, TridentCollector collector) {
    InputStream inputStream = null;
    File file = new File("fix_data.txt");
    try {
        inputStream = new BufferedInputStream(new FileInputStream(file));
        SimpleFixParser parser = new SimpleFixParser(inputStream);
        SimpleFixMessage msg = null;
        do {
            msg = parser.readFixMessage();
            if (null != msg) {
                FixMessageDto dto = new FixMessageDto();
                for (TagValue tagValue : msg.fields()) {
                    if (tagValue.tag().equals("6")) { // AvgPx
                        // dto.price = Double.valueOf((String) tagValue.value());
                        dto.price = new Double((int) (Math.random() * 100));
                    } else if (tagValue.tag().equals("35")) {
                        dto.msgType = (String) tagValue.value();
                    } else if (tagValue.tag().equals("55")) {
                        dto.symbol = (String) tagValue.value();
                    } else if (tagValue.tag().equals("11")) {
                        // dto.uid = (String) tagValue.value();
                        dto.uid = Integer.toString(uids.incrementAndGet());
                    }
                }
                new ObjectOutputStream(new ByteArrayOutputStream()).writeObject(dto);
                List<Object> message = new ArrayList<Object>();
                message.add(dto);
                collector.emit(message);
            }
        } while (msg != null);
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        IoUtils.closeSilently(inputStream);
    }
}
 

@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta, TridentCollector collector) {
    for (int i = 0; i < 10000; i++) {
        List<Object> events = new ArrayList<Object>();
        double lat = new Double(-30 + (int) (Math.random() * 75));
        double lng = new Double(-120 + (int) (Math.random() * 70));
        long time = System.currentTimeMillis();

        String diag = new Integer(320 + (int) (Math.random() * 7)).toString();
        DiagnosisEvent event = new DiagnosisEvent(lat, lng, time, diag);
        events.add(event);
        collector.emit(events);
    }
}
 

@Override
public void emitBatch(TransactionAttempt tx, Long coordinatorMeta,
                      TridentCollector collector) {
    for (int i = 0; i < 100; i++) {
        List<Object> message = new ArrayList<Object>();
        message.add(Integer.toString(i));
        collector.emit(message);
    }
}
 

@Override
public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) {
    _spout.emitBatch(tx.getTransactionId(), collector);
}
 

@Override
public void success(TransactionAttempt tx) {
    successfulTransactions.incrementAndGet();
}
 

@Override
public void success(TransactionAttempt tx) {
    successfulTransactions.incrementAndGet();
}
 

private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) {
    SimpleConsumer consumer = _connections.register(partition);
    Map ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta);
    _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long) ret.get("offset"));
    return ret;
}
 

@Override
public void success(TransactionAttempt tx) {
    _emitter.success(tx);
}
 

@Override
public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) {
    _emitter.emitBatch(tx, coordinatorMeta, collector);
}
 

@Override
public void commit(TransactionAttempt attempt) {
}
 

@Override
public void success(TransactionAttempt tx) {
}
 

@Override
public void success(TransactionAttempt tx) {
    successfulTransactions.incrementAndGet();
}
 

@Override
public void success(TransactionAttempt tx) {
    for(EmitterPartitionState state: _partitionStates.values()) {
        state.rotatingState.cleanupBefore(tx.getTransactionId());
    }            
}
 

@Override
public void success(TransactionAttempt tx) {
    _spout.ack(tx.getTransactionId());
}
 

@Override
public void success(TransactionAttempt tx) {
    successfulTransactions.incrementAndGet();
}
 

@Override
public void success(TransactionAttempt tx) {
    for (EmitterPartitionState state : _partitionStates.values()) {
        state.rotatingState.cleanupBefore(tx.getTransactionId());
    }
}