Java Code Examples for org.apache.nifi.util.StopWatch#calculateDataRate()
The following examples show how to use
org.apache.nifi.util.StopWatch#calculateDataRate() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: GetHDFS.java From localization_nifi with Apache License 2.0 | 4 votes |
|
protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) {
// process the batch of files
InputStream stream = null;
CompressionCodec codec = null;
Configuration conf = getConfiguration();
FileSystem hdfs = getFileSystem();
final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean();
final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B);
int bufferSize = bufferSizeProp != null ? bufferSizeProp.intValue() : conf.getInt(BUFFER_SIZE_KEY,
BUFFER_SIZE_DEFAULT);
final Path rootDir = new Path(context.getProperty(DIRECTORY).evaluateAttributeExpressions().getValue());
final CompressionType compressionType = CompressionType.valueOf(context.getProperty(COMPRESSION_CODEC).toString());
final boolean inferCompressionCodec = compressionType == CompressionType.AUTOMATIC;
if (inferCompressionCodec || compressionType != CompressionType.NONE) {
codec = getCompressionCodec(context, getConfiguration());
}
final CompressionCodecFactory compressionCodecFactory = new CompressionCodecFactory(conf);
for (final Path file : files) {
try {
if (!hdfs.exists(file)) {
continue; // if file is no longer there then move on
}
final String originalFilename = file.getName();
final String relativePath = getPathDifference(rootDir, file);
stream = hdfs.open(file, bufferSize);
final String outputFilename;
// Check if we should infer compression codec
if (inferCompressionCodec) {
codec = compressionCodecFactory.getCodec(file);
}
// Check if compression codec is defined (inferred or otherwise)
if (codec != null) {
stream = codec.createInputStream(stream);
outputFilename = StringUtils.removeEnd(originalFilename, codec.getDefaultExtension());
} else {
outputFilename = originalFilename;
}
FlowFile flowFile = session.create();
final StopWatch stopWatch = new StopWatch(true);
flowFile = session.importFrom(stream, flowFile);
stopWatch.stop();
final String dataRate = stopWatch.calculateDataRate(flowFile.getSize());
final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
flowFile = session.putAttribute(flowFile, CoreAttributes.PATH.key(), relativePath);
flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), outputFilename);
if (!keepSourceFiles && !hdfs.delete(file, false)) {
getLogger().warn("Could not remove {} from HDFS. Not ingesting this file ...",
new Object[]{file});
session.remove(flowFile);
continue;
}
final String transitUri = (originalFilename.startsWith("/")) ? "hdfs:/" + originalFilename : "hdfs://" + originalFilename;
session.getProvenanceReporter().receive(flowFile, transitUri);
session.transfer(flowFile, REL_SUCCESS);
getLogger().info("retrieved {} from HDFS {} in {} milliseconds at a rate of {}",
new Object[]{flowFile, file, millis, dataRate});
session.commit();
} catch (final Throwable t) {
getLogger().error("Error retrieving file {} from HDFS due to {}", new Object[]{file, t});
session.rollback();
context.yield();
} finally {
IOUtils.closeQuietly(stream);
stream = null;
}
}
}
Example 2
| Source File: GetHDFSSequenceFile.java From localization_nifi with Apache License 2.0 | 4 votes |
|
@Override
protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) {
final Configuration conf = getConfiguration();
final FileSystem hdfs = getFileSystem();
final String flowFileContentValue = context.getProperty(FLOWFILE_CONTENT).getValue();
final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean();
final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B);
if (bufferSizeProp != null) {
int bufferSize = bufferSizeProp.intValue();
conf.setInt(BUFFER_SIZE_KEY, bufferSize);
}
ComponentLog logger = getLogger();
final SequenceFileReader<Set<FlowFile>> reader;
if (flowFileContentValue.equalsIgnoreCase(VALUE_ONLY)) {
reader = new ValueReader(session);
} else {
reader = new KeyValueReader(session);
}
Set<FlowFile> flowFiles = Collections.emptySet();
for (final Path file : files) {
if (!this.isScheduled()) {
break; // This processor should stop running immediately.
}
final StopWatch stopWatch = new StopWatch(false);
try {
stopWatch.start();
if (!hdfs.exists(file)) {
continue; // If file is no longer here move on.
}
logger.debug("Reading file");
flowFiles = getFlowFiles(conf, hdfs, reader, file);
if (!keepSourceFiles && !hdfs.delete(file, false)) {
logger.warn("Unable to delete path " + file.toString() + " from HDFS. Will likely be picked up over and over...");
}
} catch (Throwable t) {
logger.error("Error retrieving file {} from HDFS due to {}", new Object[]{file, t});
session.rollback();
context.yield();
} finally {
stopWatch.stop();
long totalSize = 0;
for (FlowFile flowFile : flowFiles) {
totalSize += flowFile.getSize();
session.getProvenanceReporter().receive(flowFile, file.toString());
}
if (totalSize > 0) {
final String dataRate = stopWatch.calculateDataRate(totalSize);
final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
logger.info("Created {} flowFiles from SequenceFile {}. Ingested in {} milliseconds at a rate of {}", new Object[]{
flowFiles.size(), file.toUri().toASCIIString(), millis, dataRate});
logger.info("Transferred flowFiles {} to success", new Object[]{flowFiles});
session.transfer(flowFiles, REL_SUCCESS);
}
}
}
}
Example 3
| Source File: AbstractFlowFileServerProtocol.java From localization_nifi with Apache License 2.0 | 4 votes |
|
protected int commitTransferTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
ProcessSession session = transaction.getSession();
Set<FlowFile> flowFilesSent = transaction.getFlowFilesSent();
// we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response
CommunicationsSession commsSession = peer.getCommunicationsSession();
final Response transactionConfirmationResponse = readTransactionResponse(true, commsSession);
if (transactionConfirmationResponse.getCode() == ResponseCode.CONFIRM_TRANSACTION) {
// Confirm Checksum and echo back the confirmation.
logger.debug("{} Received {} from {}", this, transactionConfirmationResponse, peer);
final String receivedCRC = transactionConfirmationResponse.getMessage();
if (getVersionNegotiator().getVersion() > 3) {
String calculatedCRC = transaction.getCalculatedCRC();
if (!receivedCRC.equals(calculatedCRC)) {
writeTransactionResponse(true, ResponseCode.BAD_CHECKSUM, commsSession);
session.rollback();
throw new IOException(this + " Sent data to peer " + peer + " but calculated CRC32 Checksum as "
+ calculatedCRC + " while peer calculated CRC32 Checksum as " + receivedCRC
+ "; canceling transaction and rolling back session");
}
}
writeTransactionResponse(true, ResponseCode.CONFIRM_TRANSACTION, commsSession, "");
} else {
throw new ProtocolException("Expected to receive 'Confirm Transaction' response from peer " + peer + " but received " + transactionConfirmationResponse);
}
final String flowFileDescription = flowFilesSent.size() < 20 ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
final Response transactionResponse;
try {
transactionResponse = readTransactionResponse(true, commsSession);
} catch (final IOException e) {
logger.error("{} Failed to receive a response from {} when expecting a TransactionFinished Indicator."
+ " It is unknown whether or not the peer successfully received/processed the data."
+ " Therefore, {} will be rolled back, possibly resulting in data duplication of {}",
this, peer, session, flowFileDescription);
session.rollback();
throw e;
}
logger.debug("{} received {} from {}", new Object[]{this, transactionResponse, peer});
if (transactionResponse.getCode() == ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL) {
peer.penalize(port.getIdentifier(), port.getYieldPeriod(TimeUnit.MILLISECONDS));
} else if (transactionResponse.getCode() != ResponseCode.TRANSACTION_FINISHED) {
throw new ProtocolException("After sending data, expected TRANSACTION_FINISHED response but got " + transactionResponse);
}
session.commit();
StopWatch stopWatch = transaction.getStopWatch();
long bytesSent = transaction.getBytesSent();
stopWatch.stop();
final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
final String dataSize = FormatUtils.formatDataSize(bytesSent);
logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});
return flowFilesSent.size();
}
Example 4
| Source File: AbstractFlowFileServerProtocol.java From localization_nifi with Apache License 2.0 | 4 votes |
|
protected int commitReceiveTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
CommunicationsSession commsSession = peer.getCommunicationsSession();
ProcessSession session = transaction.getSession();
final Response confirmTransactionResponse = readTransactionResponse(false, commsSession);
logger.debug("{} Received {} from {}", this, confirmTransactionResponse, peer);
switch (confirmTransactionResponse.getCode()) {
case CONFIRM_TRANSACTION:
break;
case BAD_CHECKSUM:
session.rollback();
throw new IOException(this + " Received a BadChecksum response from peer " + peer);
default:
throw new ProtocolException(this + " Received unexpected Response Code from peer " + peer + " : " + confirmTransactionResponse + "; expected 'Confirm Transaction' Response Code");
}
// Commit the session so that we have persisted the data
session.commit();
if (transaction.getContext().getAvailableRelationships().isEmpty()) {
// Confirm that we received the data and the peer can now discard it but that the peer should not
// send any more data for a bit
logger.debug("{} Sending TRANSACTION_FINISHED_BUT_DESTINATION_FULL to {}", this, peer);
writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL, commsSession);
} else {
// Confirm that we received the data and the peer can now discard it
logger.debug("{} Sending TRANSACTION_FINISHED to {}", this, peer);
writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED, commsSession);
}
Set<FlowFile> flowFilesReceived = transaction.getFlowFilesSent();
long bytesReceived = transaction.getBytesSent();
StopWatch stopWatch = transaction.getStopWatch();
stopWatch.stop();
final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
final String dataSize = FormatUtils.formatDataSize(bytesReceived);
logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});
return flowFilesReceived.size();
}
Example 5
| Source File: StandardRemoteGroupPort.java From localization_nifi with Apache License 2.0 | 4 votes |
|
private int transferFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session, final FlowFile firstFlowFile) throws IOException, ProtocolException {
FlowFile flowFile = firstFlowFile;
try {
final String userDn = transaction.getCommunicant().getDistinguishedName();
final long startSendingNanos = System.nanoTime();
final StopWatch stopWatch = new StopWatch(true);
long bytesSent = 0L;
final Set<FlowFile> flowFilesSent = new HashSet<>();
boolean continueTransaction = true;
while (continueTransaction) {
final long startNanos = System.nanoTime();
// call codec.encode within a session callback so that we have the InputStream to read the FlowFile
final FlowFile toWrap = flowFile;
session.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream in) throws IOException {
final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize());
transaction.send(dataPacket);
}
});
final long transferNanos = System.nanoTime() - startNanos;
final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);
flowFilesSent.add(flowFile);
bytesSent += flowFile.getSize();
logger.debug("{} Sent {} to {}", this, flowFile, transaction.getCommunicant().getUrl());
final String transitUri = transaction.getCommunicant().createTransitUri(flowFile.getAttribute(CoreAttributes.UUID.key()));
session.getProvenanceReporter().send(flowFile, transitUri, "Remote DN=" + userDn, transferMillis, false);
session.remove(flowFile);
final long sendingNanos = System.nanoTime() - startSendingNanos;
if (sendingNanos < BATCH_SEND_NANOS) {
flowFile = session.get();
} else {
flowFile = null;
}
continueTransaction = (flowFile != null);
}
transaction.confirm();
// consume input stream entirely, ignoring its contents. If we
// don't do this, the Connection will not be returned to the pool
stopWatch.stop();
final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
final String dataSize = FormatUtils.formatDataSize(bytesSent);
transaction.complete();
session.commit();
final String flowFileDescription = (flowFilesSent.size() < 20) ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});
return flowFilesSent.size();
} catch (final Exception e) {
session.rollback();
throw e;
}
}
Example 6
| Source File: StandardRemoteGroupPort.java From localization_nifi with Apache License 2.0 | 4 votes |
|
private int receiveFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session) throws IOException, ProtocolException {
final String userDn = transaction.getCommunicant().getDistinguishedName();
final StopWatch stopWatch = new StopWatch(true);
final Set<FlowFile> flowFilesReceived = new HashSet<>();
long bytesReceived = 0L;
while (true) {
final long start = System.nanoTime();
final DataPacket dataPacket = transaction.receive();
if (dataPacket == null) {
break;
}
FlowFile flowFile = session.create();
flowFile = session.putAllAttributes(flowFile, dataPacket.getAttributes());
final Communicant communicant = transaction.getCommunicant();
final String host = StringUtils.isEmpty(communicant.getHost()) ? "unknown" : communicant.getHost();
final String port = communicant.getPort() < 0 ? "unknown" : String.valueOf(communicant.getPort());
final Map<String,String> attributes = new HashMap<>(2);
attributes.put(SiteToSiteAttributes.S2S_HOST.key(), host);
attributes.put(SiteToSiteAttributes.S2S_ADDRESS.key(), host + ":" + port);
flowFile = session.putAllAttributes(flowFile, attributes);
flowFile = session.importFrom(dataPacket.getData(), flowFile);
final long receiveNanos = System.nanoTime() - start;
flowFilesReceived.add(flowFile);
String sourceFlowFileIdentifier = dataPacket.getAttributes().get(CoreAttributes.UUID.key());
if (sourceFlowFileIdentifier == null) {
sourceFlowFileIdentifier = "<Unknown Identifier>";
}
final String transitUri = transaction.getCommunicant().createTransitUri(sourceFlowFileIdentifier);
session.getProvenanceReporter().receive(flowFile, transitUri, "urn:nifi:" + sourceFlowFileIdentifier,
"Remote DN=" + userDn, TimeUnit.NANOSECONDS.toMillis(receiveNanos));
session.transfer(flowFile, Relationship.ANONYMOUS);
bytesReceived += dataPacket.getSize();
}
// Confirm that what we received was the correct data.
transaction.confirm();
// Commit the session so that we have persisted the data
session.commit();
transaction.complete();
if (!flowFilesReceived.isEmpty()) {
stopWatch.stop();
final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
final String dataSize = FormatUtils.formatDataSize(bytesReceived);
logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});
}
return flowFilesReceived.size();
}
Example 7
| Source File: GetHDFS.java From nifi with Apache License 2.0 | 4 votes |
|
protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) {
// process the batch of files
InputStream stream = null;
CompressionCodec codec = null;
Configuration conf = getConfiguration();
FileSystem hdfs = getFileSystem();
final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean();
final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B);
int bufferSize = bufferSizeProp != null ? bufferSizeProp.intValue() : conf.getInt(BUFFER_SIZE_KEY,
BUFFER_SIZE_DEFAULT);
final Path rootDir = new Path(context.getProperty(DIRECTORY).evaluateAttributeExpressions().getValue());
final CompressionType compressionType = CompressionType.valueOf(context.getProperty(COMPRESSION_CODEC).toString());
final boolean inferCompressionCodec = compressionType == CompressionType.AUTOMATIC;
if (inferCompressionCodec || compressionType != CompressionType.NONE) {
codec = getCompressionCodec(context, getConfiguration());
}
final CompressionCodecFactory compressionCodecFactory = new CompressionCodecFactory(conf);
for (final Path file : files) {
try {
if (!getUserGroupInformation().doAs((PrivilegedExceptionAction<Boolean>) () -> hdfs.exists(file))) {
continue; // if file is no longer there then move on
}
final String originalFilename = file.getName();
final String relativePath = getPathDifference(rootDir, file);
stream = getUserGroupInformation().doAs((PrivilegedExceptionAction<FSDataInputStream>) () -> hdfs.open(file, bufferSize));
final String outputFilename;
// Check if we should infer compression codec
if (inferCompressionCodec) {
codec = compressionCodecFactory.getCodec(file);
}
// Check if compression codec is defined (inferred or otherwise)
if (codec != null) {
stream = codec.createInputStream(stream);
outputFilename = StringUtils.removeEnd(originalFilename, codec.getDefaultExtension());
} else {
outputFilename = originalFilename;
}
FlowFile flowFile = session.create();
final StopWatch stopWatch = new StopWatch(true);
flowFile = session.importFrom(stream, flowFile);
stopWatch.stop();
final String dataRate = stopWatch.calculateDataRate(flowFile.getSize());
final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
flowFile = session.putAttribute(flowFile, CoreAttributes.PATH.key(), relativePath.isEmpty() ? "." : relativePath);
flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), outputFilename);
if (!keepSourceFiles && !getUserGroupInformation().doAs((PrivilegedExceptionAction<Boolean>) () -> hdfs.delete(file, false))) {
getLogger().warn("Could not remove {} from HDFS. Not ingesting this file ...",
new Object[]{file});
session.remove(flowFile);
continue;
}
session.getProvenanceReporter().receive(flowFile, file.toString());
session.transfer(flowFile, REL_SUCCESS);
getLogger().info("retrieved {} from HDFS {} in {} milliseconds at a rate of {}",
new Object[]{flowFile, file, millis, dataRate});
session.commit();
} catch (final Throwable t) {
getLogger().error("Error retrieving file {} from HDFS due to {}", new Object[]{file, t});
session.rollback();
context.yield();
} finally {
IOUtils.closeQuietly(stream);
stream = null;
}
}
}
Example 8
| Source File: GetHDFSSequenceFile.java From nifi with Apache License 2.0 | 4 votes |
|
@Override
protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) {
final Configuration conf = getConfiguration();
final FileSystem hdfs = getFileSystem();
final String flowFileContentValue = context.getProperty(FLOWFILE_CONTENT).getValue();
final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean();
final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B);
if (bufferSizeProp != null) {
int bufferSize = bufferSizeProp.intValue();
conf.setInt(BUFFER_SIZE_KEY, bufferSize);
}
ComponentLog logger = getLogger();
final SequenceFileReader<Set<FlowFile>> reader;
if (flowFileContentValue.equalsIgnoreCase(VALUE_ONLY)) {
reader = new ValueReader(session);
} else {
reader = new KeyValueReader(session);
}
Set<FlowFile> flowFiles = Collections.emptySet();
for (final Path file : files) {
if (!this.isScheduled()) {
break; // This processor should stop running immediately.
}
final StopWatch stopWatch = new StopWatch(false);
try {
stopWatch.start();
if (!hdfs.exists(file)) {
continue; // If file is no longer here move on.
}
logger.debug("Reading file");
flowFiles = getFlowFiles(conf, hdfs, reader, file);
if (!keepSourceFiles && !hdfs.delete(file, false)) {
logger.warn("Unable to delete path " + file.toString() + " from HDFS. Will likely be picked up over and over...");
}
} catch (Throwable t) {
logger.error("Error retrieving file {} from HDFS due to {}", new Object[]{file, t});
session.rollback();
context.yield();
} finally {
stopWatch.stop();
long totalSize = 0;
for (FlowFile flowFile : flowFiles) {
totalSize += flowFile.getSize();
session.getProvenanceReporter().receive(flowFile, file.toString());
}
if (totalSize > 0) {
final String dataRate = stopWatch.calculateDataRate(totalSize);
final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
logger.info("Created {} flowFiles from SequenceFile {}. Ingested in {} milliseconds at a rate of {}", new Object[]{
flowFiles.size(), file.toUri().toASCIIString(), millis, dataRate});
logger.info("Transferred flowFiles {} to success", new Object[]{flowFiles});
session.transfer(flowFiles, REL_SUCCESS);
}
}
}
}
Example 9
| Source File: AbstractFlowFileServerProtocol.java From nifi with Apache License 2.0 | 4 votes |
|
protected int commitTransferTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
ProcessSession session = transaction.getSession();
Set<FlowFile> flowFilesSent = transaction.getFlowFilesSent();
// we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response
CommunicationsSession commsSession = peer.getCommunicationsSession();
final Response transactionConfirmationResponse = readTransactionResponse(true, commsSession);
if (transactionConfirmationResponse.getCode() == ResponseCode.CONFIRM_TRANSACTION) {
// Confirm Checksum and echo back the confirmation.
logger.debug("{} Received {} from {}", this, transactionConfirmationResponse, peer);
final String receivedCRC = transactionConfirmationResponse.getMessage();
if (getVersionNegotiator().getVersion() > 3) {
String calculatedCRC = transaction.getCalculatedCRC();
if (!receivedCRC.equals(calculatedCRC)) {
writeTransactionResponse(true, ResponseCode.BAD_CHECKSUM, commsSession);
session.rollback();
throw new IOException(this + " Sent data to peer " + peer + " but calculated CRC32 Checksum as "
+ calculatedCRC + " while peer calculated CRC32 Checksum as " + receivedCRC
+ "; canceling transaction and rolling back session");
}
}
writeTransactionResponse(true, ResponseCode.CONFIRM_TRANSACTION, commsSession, "");
} else {
throw new ProtocolException("Expected to receive 'Confirm Transaction' response from peer " + peer + " but received " + transactionConfirmationResponse);
}
final String flowFileDescription = flowFilesSent.size() < 20 ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
final Response transactionResponse;
try {
transactionResponse = readTransactionResponse(true, commsSession);
} catch (final IOException e) {
logger.error("{} Failed to receive a response from {} when expecting a TransactionFinished Indicator."
+ " It is unknown whether or not the peer successfully received/processed the data."
+ " Therefore, {} will be rolled back, possibly resulting in data duplication of {}",
this, peer, session, flowFileDescription);
session.rollback();
throw e;
}
logger.debug("{} received {} from {}", new Object[]{this, transactionResponse, peer});
if (transactionResponse.getCode() == ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL) {
peer.penalize(port.getIdentifier(), port.getYieldPeriod(TimeUnit.MILLISECONDS));
} else if (transactionResponse.getCode() != ResponseCode.TRANSACTION_FINISHED) {
throw new ProtocolException("After sending data, expected TRANSACTION_FINISHED response but got " + transactionResponse);
}
session.commit();
StopWatch stopWatch = transaction.getStopWatch();
long bytesSent = transaction.getBytesSent();
stopWatch.stop();
final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
final String dataSize = FormatUtils.formatDataSize(bytesSent);
logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});
return flowFilesSent.size();
}
Example 10
| Source File: AbstractFlowFileServerProtocol.java From nifi with Apache License 2.0 | 4 votes |
|
protected int commitReceiveTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
CommunicationsSession commsSession = peer.getCommunicationsSession();
ProcessSession session = transaction.getSession();
final Response confirmTransactionResponse = readTransactionResponse(false, commsSession);
logger.debug("{} Received {} from {}", this, confirmTransactionResponse, peer);
switch (confirmTransactionResponse.getCode()) {
case CONFIRM_TRANSACTION:
break;
case BAD_CHECKSUM:
session.rollback();
throw new IOException(this + " Received a BadChecksum response from peer " + peer);
default:
throw new ProtocolException(this + " Received unexpected Response Code from peer " + peer + " : " + confirmTransactionResponse + "; expected 'Confirm Transaction' Response Code");
}
// Commit the session so that we have persisted the data
session.commit();
if (transaction.getContext().getAvailableRelationships().isEmpty()) {
// Confirm that we received the data and the peer can now discard it but that the peer should not
// send any more data for a bit
logger.debug("{} Sending TRANSACTION_FINISHED_BUT_DESTINATION_FULL to {}", this, peer);
writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL, commsSession);
} else {
// Confirm that we received the data and the peer can now discard it
logger.debug("{} Sending TRANSACTION_FINISHED to {}", this, peer);
writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED, commsSession);
}
Set<FlowFile> flowFilesReceived = transaction.getFlowFilesSent();
long bytesReceived = transaction.getBytesSent();
StopWatch stopWatch = transaction.getStopWatch();
stopWatch.stop();
final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
final String dataSize = FormatUtils.formatDataSize(bytesReceived);
logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});
return flowFilesReceived.size();
}
Example 11
| Source File: StandardRemoteGroupPort.java From nifi with Apache License 2.0 | 4 votes |
|
private int transferFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session, final FlowFile firstFlowFile) throws IOException, ProtocolException {
FlowFile flowFile = firstFlowFile;
try {
final String userDn = transaction.getCommunicant().getDistinguishedName();
final long startSendingNanos = System.nanoTime();
final StopWatch stopWatch = new StopWatch(true);
long bytesSent = 0L;
final SiteToSiteClientConfig siteToSiteClientConfig = getSiteToSiteClient().getConfig();
final long maxBatchBytes = siteToSiteClientConfig.getPreferredBatchSize();
final int maxBatchCount = siteToSiteClientConfig.getPreferredBatchCount();
final long preferredBatchDuration = siteToSiteClientConfig.getPreferredBatchDuration(TimeUnit.NANOSECONDS);
final long maxBatchDuration = preferredBatchDuration > 0 ? preferredBatchDuration : BATCH_SEND_NANOS;
final Set<FlowFile> flowFilesSent = new HashSet<>();
boolean continueTransaction = true;
while (continueTransaction) {
final long startNanos = System.nanoTime();
// call codec.encode within a session callback so that we have the InputStream to read the FlowFile
final FlowFile toWrap = flowFile;
session.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream in) throws IOException {
final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize());
transaction.send(dataPacket);
}
});
final long transferNanos = System.nanoTime() - startNanos;
final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);
flowFilesSent.add(flowFile);
bytesSent += flowFile.getSize();
logger.debug("{} Sent {} to {}", this, flowFile, transaction.getCommunicant().getUrl());
final String transitUri = transaction.getCommunicant().createTransitUri(flowFile.getAttribute(CoreAttributes.UUID.key()));
flowFile = session.putAttribute(flowFile, SiteToSiteAttributes.S2S_PORT_ID.key(), getTargetIdentifier());
session.getProvenanceReporter().send(flowFile, transitUri, "Remote DN=" + userDn, transferMillis, false);
session.remove(flowFile);
final long sendingNanos = System.nanoTime() - startSendingNanos;
if (maxBatchCount > 0 && flowFilesSent.size() >= maxBatchCount) {
flowFile = null;
} else if (maxBatchBytes > 0 && bytesSent >= maxBatchBytes) {
flowFile = null;
} else if (sendingNanos >= maxBatchDuration) {
flowFile = null;
} else {
flowFile = session.get();
}
continueTransaction = (flowFile != null);
}
transaction.confirm();
// consume input stream entirely, ignoring its contents. If we
// don't do this, the Connection will not be returned to the pool
stopWatch.stop();
final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
final String dataSize = FormatUtils.formatDataSize(bytesSent);
transaction.complete();
session.commit();
final String flowFileDescription = (flowFilesSent.size() < 20) ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});
return flowFilesSent.size();
} catch (final Exception e) {
session.rollback();
throw e;
}
}
Example 12
| Source File: StandardRemoteGroupPort.java From nifi with Apache License 2.0 | 4 votes |
|
private int receiveFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session) throws IOException, ProtocolException {
final String userDn = transaction.getCommunicant().getDistinguishedName();
final StopWatch stopWatch = new StopWatch(true);
final Set<FlowFile> flowFilesReceived = new HashSet<>();
long bytesReceived = 0L;
while (true) {
final long start = System.nanoTime();
final DataPacket dataPacket = transaction.receive();
if (dataPacket == null) {
break;
}
FlowFile flowFile = session.create();
flowFile = session.putAllAttributes(flowFile, dataPacket.getAttributes());
final Communicant communicant = transaction.getCommunicant();
final String host = StringUtils.isEmpty(communicant.getHost()) ? "unknown" : communicant.getHost();
final String port = communicant.getPort() < 0 ? "unknown" : String.valueOf(communicant.getPort());
final Map<String,String> attributes = new HashMap<>(2);
attributes.put(SiteToSiteAttributes.S2S_HOST.key(), host);
attributes.put(SiteToSiteAttributes.S2S_ADDRESS.key(), host + ":" + port);
attributes.put(SiteToSiteAttributes.S2S_PORT_ID.key(), getTargetIdentifier());
flowFile = session.putAllAttributes(flowFile, attributes);
flowFile = session.importFrom(dataPacket.getData(), flowFile);
final long receiveNanos = System.nanoTime() - start;
flowFilesReceived.add(flowFile);
String sourceFlowFileIdentifier = dataPacket.getAttributes().get(CoreAttributes.UUID.key());
if (sourceFlowFileIdentifier == null) {
sourceFlowFileIdentifier = "<Unknown Identifier>";
}
final String transitUri = transaction.getCommunicant().createTransitUri(sourceFlowFileIdentifier);
session.getProvenanceReporter().receive(flowFile, transitUri, "urn:nifi:" + sourceFlowFileIdentifier,
"Remote DN=" + userDn, TimeUnit.NANOSECONDS.toMillis(receiveNanos));
session.transfer(flowFile, Relationship.ANONYMOUS);
bytesReceived += dataPacket.getSize();
}
// Confirm that what we received was the correct data.
transaction.confirm();
// Commit the session so that we have persisted the data
session.commit();
transaction.complete();
if (!flowFilesReceived.isEmpty()) {
stopWatch.stop();
final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
final String dataSize = FormatUtils.formatDataSize(bytesReceived);
logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});
}
return flowFilesReceived.size();
}
