org.apache.nifi.util.StopWatch Java Examples

@Override
public DeleteOperationResponse deleteById(String index, String type, List<String> ids) {
    try {
        StringBuilder sb = new StringBuilder();
        for (int idx = 0; idx < ids.size(); idx++) {
            String header = buildBulkHeader("delete", index, type, ids.get(idx));
            sb.append(header).append("\n");
        }
        HttpEntity entity = new NStringEntity(sb.toString(), ContentType.APPLICATION_JSON);
        StopWatch watch = new StopWatch();
        watch.start();
        Response response = client.performRequest("POST", "/_bulk", Collections.emptyMap(), entity);
        watch.stop();

        if (getLogger().isDebugEnabled()) {
            getLogger().debug(String.format("Response for bulk delete: %s",
                    IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)));
        }

        DeleteOperationResponse dor = new DeleteOperationResponse(watch.getDuration(TimeUnit.MILLISECONDS));

        return dor;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}
 

/**
 * Process elements children based on the parser mapping.
 * Any String values are added to attributes
 * For List, the processList method is called to iterate and process
 * For an Object this method is called recursively
 * While adding to the attributes the key is prefixed by parent
 * @param parent       parent key for this element, used as a prefix for attribute key
 * @param element      element to be processed
 * @param attributes   map of attributes to populate
 * @return             map of processed data, value can contain String or Map of Strings
 */
protected Map<String, Object> processElement(String parent, Object element, Map<String, String> attributes) {
    final StopWatch stopWatch = new StopWatch(true);

    Map<String, Object> map = new LinkedHashMap<String, Object>();
    String name = element.getClass().getName();
    Map<String, String> jexlMap = processMap.get(name); // get JEXL mappings for this element

    if (jexlMap == null) {
        getLogger().warn("Missing mapping for element " + name);
        return null;
    }

    for (Entry<String, String> entry : jexlMap.entrySet()) { // evaluate JEXL for each child element
        jexlCtx.set("element", element);
        JexlExpression jexlExpr = jexl.createExpression(entry.getValue());
        Object value = jexlExpr.evaluate(jexlCtx);
        String key = entry.getKey();
        String prefix = parent != null ? parent + "." + key : key;
        addElement(map, prefix, key, value, attributes);
    }
    stopWatch.stop();
    getLogger().debug("Processed {} in {}", new Object[] {name, stopWatch.getDuration(TimeUnit.MILLISECONDS)});

    return map;
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
    getLogger().debug("Loading packages");
    final StopWatch stopWatch = new StopWatch(true);

    // Load required MDHT packages
    System.setProperty( "org.eclipse.emf.ecore.EPackage.Registry.INSTANCE",
            "org.eclipse.emf.ecore.impl.EPackageRegistryImpl" );
    CDAPackage.eINSTANCE.eClass();
    HITSPPackage.eINSTANCE.eClass();
    CCDPackage.eINSTANCE.eClass();
    ConsolPackage.eINSTANCE.eClass();
    IHEPackage.eINSTANCE.eClass();
    stopWatch.stop();
    getLogger().debug("Loaded packages in {}", new Object[] {stopWatch.getDuration(TimeUnit.MILLISECONDS)});

    // Initialize JEXL
    jexl = new JexlBuilder().cache(1024).debug(false).silent(true).strict(false).create();
    jexlCtx = new MapContext();

    getLogger().debug("Loading mappings");
    loadMappings(); // Load CDA mappings for parser

}
 

private void rebuildIndex(final File tempIndexDir, final File migratedIndexDir) throws IOException {
    final EventIndexWriter writer = indexManager.borrowIndexWriter(tempIndexDir);

    try {
        final EventIterator eventIterator = eventStore.getEventsByTimestamp(minTimestamp, maxTimestamp);

        final StopWatch stopWatch = new StopWatch(true);

        Optional<ProvenanceEventRecord> optionalEvent;
        while ((optionalEvent = eventIterator.nextEvent()).isPresent()) {
            final ProvenanceEventRecord event = optionalEvent.get();

            final Document document = eventConverter.convert(event, event.getEventId());
            writer.index(document, Integer.MAX_VALUE);
            successCount++;
        }

        writer.commit();
        stopWatch.stop();
        logger.info("Successfully indexed {} events to {} in {}", successCount, tempIndexDir, stopWatch.getDuration());
    } finally {
        indexManager.returnIndexWriter(writer, true, true);
    }

    Files.move(tempIndexDir.toPath(), migratedIndexDir.toPath());
}
 

private void writeFlowFiles(PartitionContext context, Iterable<EventData> messages, ProcessSession session, StopWatch stopWatch) {
    final String eventHubName = context.getEventHubPath();
    final String partitionId = context.getPartitionId();
    final String consumerGroup = context.getConsumerGroupName();
    messages.forEach(eventData -> {
        FlowFile flowFile = session.create();

        final Map<String, String> attributes = new HashMap<>();
        putEventHubAttributes(attributes, eventHubName, partitionId, eventData);

        flowFile = session.putAllAttributes(flowFile, attributes);
        flowFile = session.write(flowFile, out -> {
            out.write(eventData.getBytes());
        });

        transferTo(REL_SUCCESS, session, stopWatch, eventHubName, partitionId, consumerGroup, flowFile);
    });
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final StopWatch stopWatch = new StopWatch(true);
        final byte[] buffer = new byte[(int) flowFile.getSize()];
        session.read(flowFile, in -> StreamUtils.fillBuffer(in, buffer));

        try {
            sendMessage(buffer);
        } catch (final ProcessException processException) {
            getLogger().error("Failed to send {} to EventHub due to {}; routing to failure", new Object[]{flowFile, processException}, processException);
            session.transfer(session.penalize(flowFile), REL_FAILURE);
            return;
        }

        final String namespace = context.getProperty(NAMESPACE).getValue();
        final String eventHubName = context.getProperty(EVENT_HUB_NAME).getValue();
        session.getProvenanceReporter().send(flowFile, "amqps://" + namespace + ".servicebus.windows.net" + "/" + eventHubName, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);

}
 

@Override
public void onEvents(PartitionContext context, Iterable<EventData> messages) throws Exception {
    final ProcessSession session = processSessionFactory.createSession();

    try {

        final StopWatch stopWatch = new StopWatch(true);

        if (readerFactory != null && writerFactory != null) {
            writeRecords(context, messages, session, stopWatch);
        } else {
            writeFlowFiles(context, messages, session, stopWatch);
        }

        // Commit NiFi first.
        session.commit();
        // If creating an Event Hub checkpoint failed, then the same message can be retrieved again.
        context.checkpoint();

    } catch (Exception e) {
        getLogger().error("Unable to fully process received message due to " + e, e);
        // FlowFiles those are already committed will not get rollback.
        session.rollback();
    }

}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    try {
        populateSenderQueue(context);
    } catch (ProcessException e) {
        context.yield();
        throw e;
    }

    final StopWatch stopWatch = new StopWatch(true);

    final String partitioningKeyAttributeName = context.getProperty(PARTITIONING_KEY_ATTRIBUTE_NAME).getValue();

    // Get N flow files
    final int maxBatchSize = NumberUtils.toInt(context.getProperty(MAX_BATCH_SIZE).getValue(), 100);
    final List<FlowFile> flowFileList = session.get(maxBatchSize);

    // Convert and send each flow file
    final BlockingQueue<CompletableFuture<FlowFileResultCarrier<Relationship>>> futureQueue = new LinkedBlockingQueue<>();
    for (FlowFile flowFile : flowFileList) {
        if (flowFile == null) {
            continue;
        }

        futureQueue.offer(handleFlowFile(flowFile, partitioningKeyAttributeName, session));
    }

    waitForAllFutures(context, session, stopWatch, futureQueue);
}
 

public FlowFileTransaction(ProcessSession session, ProcessContext context, StopWatch stopWatch, long bytesSent, Set<FlowFile> flowFilesSent, String calculatedCRC) {
    this.session = session;
    this.context = context;
    this.stopWatch = stopWatch;
    this.bytesSent = bytesSent;
    this.flowFilesSent = flowFilesSent;
    this.calculatedCRC = calculatedCRC;
}
 

@Override
public IndexOperationResponse bulk(List<IndexOperationRequest> operations) {
    try {
        StringBuilder payload = new StringBuilder();
        for (int index = 0; index < operations.size(); index++) {
            IndexOperationRequest or = operations.get(index);
            buildRequest(or, payload);
        }

        if (getLogger().isDebugEnabled()) {
            getLogger().debug(payload.toString());
        }
        HttpEntity entity = new NStringEntity(payload.toString(), ContentType.APPLICATION_JSON);
        StopWatch watch = new StopWatch();
        watch.start();
        Response response = client.performRequest("POST", "/_bulk", Collections.emptyMap(), entity);
        watch.stop();

        String rawResponse = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);

        if (getLogger().isDebugEnabled()) {
            getLogger().debug(String.format("Response was: %s", rawResponse));
        }

        IndexOperationResponse retVal = IndexOperationResponse.fromJsonResponse(rawResponse);

        return retVal;
    } catch (Exception ex) {
        throw new ElasticsearchError(ex);
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    updateMapping(context);
    final List<FlowFile> flowFiles = session.get(5);
    if (flowFiles.isEmpty()) {
        return;
    }

    final ComponentLog logger = getLogger();

    final int maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();

    for (FlowFile flowFile : flowFiles) {
        if (flowFile.getSize() > maxBufferSize) {
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }

        final StopWatch stopWatch = new StopWatch(true);

        flowFile = session.write(flowFile, new ReplaceTextCallback(context, flowFile, maxBufferSize));

        logger.info("Transferred {} to 'success'", new Object[]{flowFile});
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    }
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
    final String dbFileString = context.getProperty(GEO_DATABASE_FILE).getValue();
    final File dbFile = new File(dbFileString);
    final StopWatch stopWatch = new StopWatch(true);
    final DatabaseReader reader = new DatabaseReader.Builder(dbFile).build();
    stopWatch.stop();
    getLogger().info("Completed loading of Maxmind Geo Database.  Elapsed time was {} milliseconds.", new Object[]{stopWatch.getDuration(TimeUnit.MILLISECONDS)});
    databaseReaderRef.set(reader);
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
    final String dbFileString = context.getProperty(GEO_DATABASE_FILE).evaluateAttributeExpressions().getValue();
    final File dbFile = new File(dbFileString);
    final StopWatch stopWatch = new StopWatch(true);
    final DatabaseReader reader = new DatabaseReader.Builder(dbFile).build();
    stopWatch.stop();
    getLogger().info("Completed loading of Maxmind Database.  Elapsed time was {} milliseconds.", new Object[]{stopWatch.getDuration(TimeUnit.MILLISECONDS)});
    databaseReaderRef.set(reader);
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    updateMapping(context);
    final List<FlowFile> flowFiles = session.get(5);
    if (flowFiles.isEmpty()) {
        return;
    }

    final ComponentLog logger = getLogger();

    final int maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();

    for (FlowFile flowFile : flowFiles) {
        if (flowFile.getSize() > maxBufferSize) {
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }

        final StopWatch stopWatch = new StopWatch(true);

        flowFile = session.write(flowFile, new ReplaceTextCallback(context, flowFile, maxBufferSize));

        logger.info("Transferred {} to 'success'", new Object[]{flowFile});
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    }
}
 

private void fetchListing(final ProcessContext context, final ProcessSession session, final FileTransfer transfer) throws IOException {
    BlockingQueue<FileInfo> queue = fileQueueRef.get();
    if (queue == null) {
        final boolean useNaturalOrdering = context.getProperty(FileTransfer.USE_NATURAL_ORDERING).asBoolean();
        queue = useNaturalOrdering ? new PriorityBlockingQueue<FileInfo>(25000) : new LinkedBlockingQueue<FileInfo>(25000);
        fileQueueRef.set(queue);
    }

    final StopWatch stopWatch = new StopWatch(true);
    final List<FileInfo> listing = transfer.getListing();
    final long millis = stopWatch.getElapsed(TimeUnit.MILLISECONDS);

    int newItems = 0;
    mutuallyExclusiveTransferLock.lock();
    try {
        for (final FileInfo file : listing) {
            if (!queue.contains(file) && !processing.contains(file)) {
                if (!queue.offer(file)) {
                    break;
                }
                newItems++;
            }
        }
    } finally {
        mutuallyExclusiveTransferLock.unlock();
    }

    getLogger().info("Obtained file listing in {} milliseconds; listing had {} items, {} of which were new",
            new Object[]{millis, listing.size(), newItems});
}
 

private void fetchListing(final ProcessContext context, final ProcessSession session, final FileTransfer transfer) throws IOException {
    BlockingQueue<FileInfo> queue = fileQueueRef.get();
    if (queue == null) {
        final boolean useNaturalOrdering = context.getProperty(FileTransfer.USE_NATURAL_ORDERING).asBoolean();
        queue = useNaturalOrdering ? new PriorityBlockingQueue<FileInfo>(25000) : new LinkedBlockingQueue<FileInfo>(25000);
        fileQueueRef.set(queue);
    }

    final StopWatch stopWatch = new StopWatch(true);
    final List<FileInfo> listing = transfer.getListing();
    final long millis = stopWatch.getElapsed(TimeUnit.MILLISECONDS);

    int newItems = 0;
    mutuallyExclusiveTransferLock.lock();
    try {
        for (final FileInfo file : listing) {
            if (!queue.contains(file) && !processing.contains(file)) {
                if (!queue.offer(file)) {
                    break;
                }
                newItems++;
            }
        }
    } finally {
        mutuallyExclusiveTransferLock.unlock();
    }

    getLogger().info("Obtained file listing in {} milliseconds; listing had {} items, {} of which were new",
            new Object[]{millis, listing.size(), newItems});
}
 

/**
 * event handler method to handle the FlowFile being forwarded to the Processor by the framework. The FlowFile contents is sent out as a UDP datagram using an acquired ChannelSender object. If the
 * FlowFile contents was sent out successfully then the FlowFile is forwarded to the success relationship. If an error occurred then the FlowFile is forwarded to the failure relationship.
 *
 * @param context
 *            - the current process context.
 *
 * @param sessionFactory
 *            - a factory object to obtain a process session.
 */
@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
    final ProcessSession session = sessionFactory.createSession();
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        pruneIdleSenders(context.getProperty(IDLE_EXPIRATION).asTimePeriod(TimeUnit.MILLISECONDS).longValue());
        context.yield();
        return;
    }

    ChannelSender sender = acquireSender(context, session, flowFile);
    if (sender == null) {
        return;
    }

    try {
        byte[] content = readContent(session, flowFile);
        StopWatch stopWatch = new StopWatch(true);
        sender.send(content);
        session.getProvenanceReporter().send(flowFile, transitUri, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
        session.commit();
    } catch (Exception e) {
        getLogger().error("Exception while handling a process session, transferring {} to failure.", new Object[] { flowFile }, e);
        onFailure(context, session, flowFile);
    } finally {
        relinquishSender(sender);
    }
}
 

/**
 * event handler method to handle the FlowFile being forwarded to the Processor by the framework. The FlowFile contents is sent out as a UDP datagram using an acquired ChannelSender object. If the
 * FlowFile contents was sent out successfully then the FlowFile is forwarded to the success relationship. If an error occurred then the FlowFile is forwarded to the failure relationship.
 *
 * @param context
 *            - the current process context.
 *
 * @param sessionFactory
 *            - a factory object to obtain a process session.
 */
@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
    final ProcessSession session = sessionFactory.createSession();
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        final PruneResult result = pruneIdleSenders(context.getProperty(IDLE_EXPIRATION).asTimePeriod(TimeUnit.MILLISECONDS).longValue());
        // yield if we closed an idle connection, or if there were no connections in the first place
        if (result.getNumClosed() > 0 || (result.getNumClosed() == 0 && result.getNumConsidered() == 0)) {
            context.yield();
        }
        return;
    }

    ChannelSender sender = acquireSender(context, session, flowFile);
    if (sender == null) {
        return;
    }

    try {
        byte[] content = readContent(session, flowFile);
        StopWatch stopWatch = new StopWatch(true);
        sender.send(content);
        session.getProvenanceReporter().send(flowFile, transitUri, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
        session.commit();
    } catch (Exception e) {
        getLogger().error("Exception while handling a process session, transferring {} to failure.", new Object[] { flowFile }, e);
        onFailure(context, session, flowFile);
    } finally {
        relinquishSender(sender);
    }
}
 

/**
 * event handler method to handle the FlowFile being forwarded to the Processor by the framework. The FlowFile contents is sent out over a TCP connection using an acquired ChannelSender object. If
 * the FlowFile contents was sent out successfully then the FlowFile is forwarded to the success relationship. If an error occurred then the FlowFile is forwarded to the failure relationship.
 *
 * @param context
 *            - the current process context.
 *
 * @param sessionFactory
 *            - a factory object to obtain a process session.
 */
@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
    final ProcessSession session = sessionFactory.createSession();
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        pruneIdleSenders(context.getProperty(IDLE_EXPIRATION).asTimePeriod(TimeUnit.MILLISECONDS).longValue());
        context.yield();
        return;
    }

    ChannelSender sender = acquireSender(context, session, flowFile);
    if (sender == null) {
        return;
    }

    try {
        String outgoingMessageDelimiter = getOutgoingMessageDelimiter(context, flowFile);
        ByteArrayOutputStream content = readContent(session, flowFile);
        if (outgoingMessageDelimiter != null) {
            Charset charset = Charset.forName(context.getProperty(CHARSET).getValue());
            content = appendDelimiter(content, outgoingMessageDelimiter, charset);
        }
        StopWatch stopWatch = new StopWatch(true);
        sender.send(content.toByteArray());
        session.getProvenanceReporter().send(flowFile, transitUri, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
        session.commit();
    } catch (Exception e) {
        onFailure(context, session, flowFile);
        getLogger().error("Exception while handling a process session, transferring {} to failure.", new Object[] { flowFile }, e);
    } finally {
        // If we are going to use this sender again, then relinquish it back to the pool.
        if (!isConnectionPerFlowFile(context)) {
            relinquishSender(sender);
        } else {
            sender.close();
        }
    }
}
 

private void loadDatabase(final File dbFile, final String dbFileChecksum) throws IOException {
    final StopWatch stopWatch = new StopWatch(true);
    final DatabaseReader reader = new DatabaseReader.Builder(dbFile).build();
    stopWatch.stop();
    getLogger().info("Completed loading of Maxmind Database.  Elapsed time was {} milliseconds.", new Object[]{stopWatch.getDuration(TimeUnit.MILLISECONDS)});
    databaseReader = reader;
    databaseChecksum = dbFileChecksum;
}
 

public FlowFileTransaction(ProcessSession session, ProcessContext context, StopWatch stopWatch, long bytesSent, Set<FlowFile> flowFilesSent, String calculatedCRC) {
    this.session = session;
    this.context = context;
    this.stopWatch = stopWatch;
    this.bytesSent = bytesSent;
    this.flowFilesSent = flowFilesSent;
    this.calculatedCRC = calculatedCRC;
}
 

protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) {
    // process the batch of files
    InputStream stream = null;
    CompressionCodec codec = null;
    Configuration conf = getConfiguration();
    FileSystem hdfs = getFileSystem();
    final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean();
    final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B);
    int bufferSize = bufferSizeProp != null ? bufferSizeProp.intValue() : conf.getInt(BUFFER_SIZE_KEY,
            BUFFER_SIZE_DEFAULT);
    final Path rootDir = new Path(context.getProperty(DIRECTORY).evaluateAttributeExpressions().getValue());

    final CompressionType compressionType = CompressionType.valueOf(context.getProperty(COMPRESSION_CODEC).toString());
    final boolean inferCompressionCodec = compressionType == CompressionType.AUTOMATIC;
    if (inferCompressionCodec || compressionType != CompressionType.NONE) {
        codec = getCompressionCodec(context, getConfiguration());
    }
    final CompressionCodecFactory compressionCodecFactory = new CompressionCodecFactory(conf);
    for (final Path file : files) {
        try {
            if (!getUserGroupInformation().doAs((PrivilegedExceptionAction<Boolean>) () -> hdfs.exists(file))) {
                continue; // if file is no longer there then move on
            }
            final String originalFilename = file.getName();
            final String relativePath = getPathDifference(rootDir, file);

            stream = getUserGroupInformation().doAs((PrivilegedExceptionAction<FSDataInputStream>) () -> hdfs.open(file, bufferSize));

            final String outputFilename;
            // Check if we should infer compression codec
            if (inferCompressionCodec) {
                codec = compressionCodecFactory.getCodec(file);
            }
            // Check if compression codec is defined (inferred or otherwise)
            if (codec != null) {
                stream = codec.createInputStream(stream);
                outputFilename = StringUtils.removeEnd(originalFilename, codec.getDefaultExtension());
            } else {
                outputFilename = originalFilename;
            }

            FlowFile flowFile = session.create();

            final StopWatch stopWatch = new StopWatch(true);
            flowFile = session.importFrom(stream, flowFile);
            stopWatch.stop();
            final String dataRate = stopWatch.calculateDataRate(flowFile.getSize());
            final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);

            flowFile = session.putAttribute(flowFile, CoreAttributes.PATH.key(), relativePath.isEmpty() ? "." : relativePath);
            flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), outputFilename);

            if (!keepSourceFiles && !getUserGroupInformation().doAs((PrivilegedExceptionAction<Boolean>) () -> hdfs.delete(file, false))) {
                getLogger().warn("Could not remove {} from HDFS. Not ingesting this file ...",
                        new Object[]{file});
                session.remove(flowFile);
                continue;
            }

            session.getProvenanceReporter().receive(flowFile, file.toString());
            session.transfer(flowFile, REL_SUCCESS);
            getLogger().info("retrieved {} from HDFS {} in {} milliseconds at a rate of {}",
                    new Object[]{flowFile, file, millis, dataRate});
            session.commit();
        } catch (final Throwable t) {
            getLogger().error("Error retrieving file {} from HDFS due to {}", new Object[]{file, t});
            session.rollback();
            context.yield();
        } finally {
            IOUtils.closeQuietly(stream);
            stream = null;
        }
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final BlockingQueue<String> partitionIds = this.partitionNames;
    final String partitionId = partitionIds.poll();
    if (partitionId == null) {
        getLogger().debug("No partitions available");
        return;
    }

    final StopWatch stopWatch = new StopWatch(true);
    try {

        final Iterable<EventData> receivedEvents = receiveEvents(context, partitionId);
        if (receivedEvents == null) {
            return;
        }

        for (final EventData eventData : receivedEvents) {
            if (null != eventData) {

                final Map<String, String> attributes = new HashMap<>();
                FlowFile flowFile = session.create();
                EventData.SystemProperties systemProperties = eventData.getSystemProperties();

                if (null != systemProperties) {
                    attributes.put("eventhub.enqueued.timestamp", String.valueOf(eventData.getSystemProperties().getEnqueuedTime()));
                    attributes.put("eventhub.offset", eventData.getSystemProperties().getOffset());
                    attributes.put("eventhub.sequence", String.valueOf(eventData.getSystemProperties().getSequenceNumber()));
                }

                attributes.put("eventhub.name", context.getProperty(EVENT_HUB_NAME).getValue());
                attributes.put("eventhub.partition", partitionId);


                flowFile = session.putAllAttributes(flowFile, attributes);
                flowFile = session.write(flowFile, out -> {
                    out.write(eventData.getBody());
                });


                session.transfer(flowFile, REL_SUCCESS);

                final String namespace = context.getProperty(NAMESPACE).getValue();
                final String eventHubName = context.getProperty(EVENT_HUB_NAME).getValue();
                final String consumerGroup = context.getProperty(CONSUMER_GROUP).getValue();
                final String transitUri = "amqps://" + namespace + ".servicebus.windows.net" + "/" + eventHubName + "/ConsumerGroups/" + consumerGroup + "/Partitions/" + partitionId;
                session.getProvenanceReporter().receive(flowFile, transitUri, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
            }
        }
    } finally {
        partitionIds.offer(partitionId);
    }
}
 

private int receiveFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session) throws IOException, ProtocolException {
    final String userDn = transaction.getCommunicant().getDistinguishedName();

    final StopWatch stopWatch = new StopWatch(true);
    final Set<FlowFile> flowFilesReceived = new HashSet<>();
    long bytesReceived = 0L;

    while (true) {
        final long start = System.nanoTime();
        final DataPacket dataPacket = transaction.receive();
        if (dataPacket == null) {
            break;
        }

        FlowFile flowFile = session.create();
        flowFile = session.putAllAttributes(flowFile, dataPacket.getAttributes());

        final Communicant communicant = transaction.getCommunicant();
        final String host = StringUtils.isEmpty(communicant.getHost()) ? "unknown" : communicant.getHost();
        final String port = communicant.getPort() < 0 ? "unknown" : String.valueOf(communicant.getPort());

        final Map<String,String> attributes = new HashMap<>(2);
        attributes.put(SiteToSiteAttributes.S2S_HOST.key(), host);
        attributes.put(SiteToSiteAttributes.S2S_ADDRESS.key(), host + ":" + port);
        attributes.put(SiteToSiteAttributes.S2S_PORT_ID.key(), getTargetIdentifier());

        flowFile = session.putAllAttributes(flowFile, attributes);

        flowFile = session.importFrom(dataPacket.getData(), flowFile);
        final long receiveNanos = System.nanoTime() - start;
        flowFilesReceived.add(flowFile);

        String sourceFlowFileIdentifier = dataPacket.getAttributes().get(CoreAttributes.UUID.key());
        if (sourceFlowFileIdentifier == null) {
            sourceFlowFileIdentifier = "<Unknown Identifier>";
        }

        final String transitUri = transaction.getCommunicant().createTransitUri(sourceFlowFileIdentifier);
        session.getProvenanceReporter().receive(flowFile, transitUri, "urn:nifi:" + sourceFlowFileIdentifier,
                "Remote DN=" + userDn, TimeUnit.NANOSECONDS.toMillis(receiveNanos));

        session.transfer(flowFile, Relationship.ANONYMOUS);
        bytesReceived += dataPacket.getSize();
    }

    // Confirm that what we received was the correct data.
    transaction.confirm();

    // Commit the session so that we have persisted the data
    session.commit();

    transaction.complete();

    if (!flowFilesReceived.isEmpty()) {
        stopWatch.stop();
        final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
        final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
        final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
        final String dataSize = FormatUtils.formatDataSize(bytesReceived);
        logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
            this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});
    }

    return flowFilesReceived.size();
}
 

private int transferFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session, final FlowFile firstFlowFile) throws IOException, ProtocolException {
    FlowFile flowFile = firstFlowFile;

    try {
        final String userDn = transaction.getCommunicant().getDistinguishedName();
        final long startSendingNanos = System.nanoTime();
        final StopWatch stopWatch = new StopWatch(true);
        long bytesSent = 0L;

        final SiteToSiteClientConfig siteToSiteClientConfig = getSiteToSiteClient().getConfig();
        final long maxBatchBytes = siteToSiteClientConfig.getPreferredBatchSize();
        final int maxBatchCount = siteToSiteClientConfig.getPreferredBatchCount();
        final long preferredBatchDuration = siteToSiteClientConfig.getPreferredBatchDuration(TimeUnit.NANOSECONDS);
        final long maxBatchDuration = preferredBatchDuration > 0 ? preferredBatchDuration : BATCH_SEND_NANOS;


        final Set<FlowFile> flowFilesSent = new HashSet<>();
        boolean continueTransaction = true;
        while (continueTransaction) {
            final long startNanos = System.nanoTime();
            // call codec.encode within a session callback so that we have the InputStream to read the FlowFile
            final FlowFile toWrap = flowFile;
            session.read(flowFile, new InputStreamCallback() {
                @Override
                public void process(final InputStream in) throws IOException {
                    final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize());
                    transaction.send(dataPacket);
                }
            });

            final long transferNanos = System.nanoTime() - startNanos;
            final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);

            flowFilesSent.add(flowFile);
            bytesSent += flowFile.getSize();
            logger.debug("{} Sent {} to {}", this, flowFile, transaction.getCommunicant().getUrl());

            final String transitUri = transaction.getCommunicant().createTransitUri(flowFile.getAttribute(CoreAttributes.UUID.key()));
            flowFile = session.putAttribute(flowFile, SiteToSiteAttributes.S2S_PORT_ID.key(), getTargetIdentifier());
            session.getProvenanceReporter().send(flowFile, transitUri, "Remote DN=" + userDn, transferMillis, false);
            session.remove(flowFile);

            final long sendingNanos = System.nanoTime() - startSendingNanos;

            if (maxBatchCount > 0 && flowFilesSent.size() >= maxBatchCount) {
                flowFile = null;
            } else if (maxBatchBytes > 0 && bytesSent >= maxBatchBytes) {
                flowFile = null;
            } else if (sendingNanos >= maxBatchDuration) {
                flowFile = null;
            } else {
                flowFile = session.get();
            }

            continueTransaction = (flowFile != null);
        }

        transaction.confirm();

        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        stopWatch.stop();
        final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
        final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
        final String dataSize = FormatUtils.formatDataSize(bytesSent);

        transaction.complete();
        session.commit();

        final String flowFileDescription = (flowFilesSent.size() < 20) ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
        logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
            this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});

        return flowFilesSent.size();
    } catch (final Exception e) {
        session.rollback();
        throw e;
    }

}
 

@Override
public void onTrigger(final ProcessContext context, ProcessSession session) throws ProcessException {
    final FlowFile original = session.get();
    if (original == null) {
        return;
    }

    final ComponentLog logger = getLogger();
    final StopWatch stopWatch = new StopWatch(true);

    final Object inputJson;
    try (final InputStream in = session.read(original)) {
        inputJson = JsonUtils.jsonToObject(in);
    } catch (final Exception e) {
        logger.error("Failed to transform {}; routing to failure", new Object[] {original, e});
        session.transfer(original, REL_FAILURE);
        return;
    }

    final String jsonString;
    final ClassLoader originalContextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        final JoltTransform transform = getTransform(context, original);
        if (customClassLoader != null) {
            Thread.currentThread().setContextClassLoader(customClassLoader);
        }

        final Object transformedJson = TransformUtils.transform(transform,inputJson);
        jsonString = context.getProperty(PRETTY_PRINT).asBoolean() ? JsonUtils.toPrettyJsonString(transformedJson) : JsonUtils.toJsonString(transformedJson);
    } catch (final Exception ex) {
        logger.error("Unable to transform {} due to {}", new Object[] {original, ex.toString(), ex});
        session.transfer(original, REL_FAILURE);
        return;
    } finally {
        if (customClassLoader != null && originalContextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(originalContextClassLoader);
        }
    }

    FlowFile transformed = session.write(original, new OutputStreamCallback() {
        @Override
        public void process(OutputStream out) throws IOException {
            out.write(jsonString.getBytes(DEFAULT_CHARSET));
        }
    });

    final String transformType = context.getProperty(JOLT_TRANSFORM).getValue();
    transformed = session.putAttribute(transformed, CoreAttributes.MIME_TYPE.key(), "application/json");
    session.transfer(transformed, REL_SUCCESS);
    session.getProvenanceReporter().modifyContent(transformed,"Modified With " + transformType ,stopWatch.getElapsed(TimeUnit.MILLISECONDS));
    logger.info("Transformed {}", new Object[]{original});
}
 

public StopWatch getStopWatch() {
    return stopWatch;
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();

    boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
    String encoding = context.getProperty(ENCODING).getValue();
    StreamCallback encoder = null;

    // Select the encoder/decoder to use
    if (encode) {
        if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
            encoder = new EncodeBase64();
        } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
            encoder = new EncodeBase32();
        } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
            encoder = new EncodeHex();
        }
    } else {
        if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
            encoder = new DecodeBase64();
        } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
            encoder = new DecodeBase32();
        } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
            encoder = new DecodeHex();
        }
    }

    if (encoder == null) {
        logger.warn("Unknown operation: {} {}", new Object[]{encode ? "encode" : "decode", encoding});
        return;
    }

    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, encoder);

        logger.info("Successfully {} {}", new Object[]{encode ? "encoded" : "decoded", flowFile});
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (Exception e) {
        logger.error("Failed to {} {} due to {}", new Object[]{encode ? "encode" : "decode", flowFile, e});
        session.transfer(flowFile, REL_FAILURE);
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();

    final Charset inputCharset = Charset.forName(context.getProperty(INPUT_CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    final Charset outputCharset = Charset.forName(context.getProperty(OUTPUT_CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    final CharBuffer charBuffer = CharBuffer.allocate(MAX_BUFFER_SIZE);

    final CharsetDecoder decoder = inputCharset.newDecoder();
    decoder.onMalformedInput(CodingErrorAction.REPLACE);
    decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
    decoder.replaceWith("?");

    final CharsetEncoder encoder = outputCharset.newEncoder();
    encoder.onMalformedInput(CodingErrorAction.REPLACE);
    encoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
    encoder.replaceWith("?".getBytes(outputCharset));

    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, new StreamCallback() {
            @Override
            public void process(final InputStream rawIn, final OutputStream rawOut) throws IOException {
                try (final BufferedReader reader = new BufferedReader(new InputStreamReader(rawIn, decoder), MAX_BUFFER_SIZE);
                        final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(rawOut, encoder), MAX_BUFFER_SIZE)) {
                    int charsRead;
                    while ((charsRead = reader.read(charBuffer)) != -1) {
                        charBuffer.flip();
                        writer.write(charBuffer.array(), 0, charsRead);
                    }

                    writer.flush();
                }
            }
        });

        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        logger.info("successfully converted characters from {} to {} for {}",
                new Object[]{inputCharset, outputCharset, flowFile});
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final Exception e) {
        throw new ProcessException(e);
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final DatabaseReader dbReader = databaseReaderRef.get();
    final String ipAttributeName = context.getProperty(IP_ADDRESS_ATTRIBUTE).evaluateAttributeExpressions(flowFile).getValue();
    final String ipAttributeValue = flowFile.getAttribute(ipAttributeName);

    if (StringUtils.isEmpty(ipAttributeName)) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("FlowFile '{}' attribute '{}' was empty. Routing to failure",
                new Object[]{flowFile, IP_ADDRESS_ATTRIBUTE.getDisplayName()});
        return;
    }

    InetAddress inetAddress = null;
    CityResponse response = null;

    try {
        inetAddress = InetAddress.getByName(ipAttributeValue);
    } catch (final IOException ioe) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Could not resolve the IP for value '{}', contained within the attribute '{}' in " +
                        "FlowFile '{}'. This is usually caused by issue resolving the appropriate DNS record or " +
                        "providing the processor with an invalid IP address ",
                        new Object[]{ipAttributeValue, IP_ADDRESS_ATTRIBUTE.getDisplayName(), flowFile}, ioe);
        return;
    }

    final StopWatch stopWatch = new StopWatch(true);
    try {
        response = dbReader.city(inetAddress);
        stopWatch.stop();
    } catch (final IOException ex) {
        // Note IOException is captured again as dbReader also makes InetAddress.getByName() calls.
        // Most name or IP resolutions failure should have been triggered in the try loop above but
        // environmental conditions may trigger errors during the second resolution as well.
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Failure while trying to find enrichment data for {} due to {}", new Object[]{flowFile, ex}, ex);
        return;
    }

    if (response == null) {
        session.transfer(flowFile, REL_NOT_FOUND);
        return;
    }

    final Map<String, String> attrs = new HashMap<>();
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.lookup.micros").toString(), String.valueOf(stopWatch.getDuration(TimeUnit.MICROSECONDS)));
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.city").toString(), response.getCity().getName());

    final Double latitude = response.getLocation().getLatitude();
    if (latitude != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.latitude").toString(), latitude.toString());
    }

    final Double longitude = response.getLocation().getLongitude();
    if (longitude != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.longitude").toString(), longitude.toString());
    }

    final Integer accuracy = response.getLocation().getAccuracyRadius();
    if (accuracy != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".accuracy").toString(), String.valueOf(accuracy));
    }

    int i = 0;
    for (final Subdivision subd : response.getSubdivisions()) {
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.subdivision.").append(i).toString(), subd.getName());
        attrs.put(new StringBuilder(ipAttributeName).append(".geo.subdivision.isocode.").append(i).toString(), subd.getIsoCode());
        i++;
    }
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.country").toString(), response.getCountry().getName());
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.country.isocode").toString(), response.getCountry().getIsoCode());
    attrs.put(new StringBuilder(ipAttributeName).append(".geo.postalcode").toString(), response.getPostal().getCode());
    flowFile = session.putAllAttributes(flowFile, attrs);

    session.transfer(flowFile, REL_FOUND);
}