org.apache.nifi.processor.ProcessContext Java Examples

@Override
protected Tuple<Map<String, RecordPath>, RecordPath> getFlowFileContext(final FlowFile flowFile, final ProcessContext context) {
    final Map<String, RecordPath> recordPaths = new HashMap<>();
    for (final PropertyDescriptor prop : context.getProperties().keySet()) {
        if (!prop.isDynamic()) {
            continue;
        }

        final String pathText = context.getProperty(prop).evaluateAttributeExpressions(flowFile).getValue();
        final RecordPath lookupRecordPath = recordPathCache.getCompiled(pathText);
        recordPaths.put(prop.getName(), lookupRecordPath);
    }

    final RecordPath resultRecordPath;
    if (context.getProperty(RESULT_RECORD_PATH).isSet()) {
        final String resultPathText = context.getProperty(RESULT_RECORD_PATH).evaluateAttributeExpressions(flowFile).getValue();
        resultRecordPath = recordPathCache.getCompiled(resultPathText);
    } else {
        resultRecordPath = null;
    }

    return new Tuple<>(recordPaths, resultRecordPath);
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) throws ProcessException {
    this.receiveBufferSize = context.getProperty(RECEIVE_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
    this.originalServerAddressList = context.getProperty(ENDPOINT_LIST).getValue();
    this.endOfMessageByte = ((byte) context.getProperty(END_OF_MESSAGE_BYTE).asInteger().intValue());
    this.connectionAttemptCount = context.getProperty(CONNECTION_ATTEMPT_COUNT).asInteger();
    this.reconnectInterval = context.getProperty(RECONNECT_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS);

    this.clientScheduler = new ScheduledThreadPoolExecutor(originalServerAddressList.split(",").length + 1);
    this.clientScheduler.setKeepAliveTime(10, TimeUnit.SECONDS);
    this.clientScheduler.allowCoreThreadTimeOut(true);

    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        final PropertyDescriptor descriptor = entry.getKey();
        if (descriptor.isDynamic()) {
            this.dynamicAttributes.put(descriptor.getName(), entry.getValue());
        }
    }
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) {
    fileFilterRef.set(createFileFilter(context));
    includeFileAttributes = context.getProperty(INCLUDE_FILE_ATTRIBUTES).asBoolean();

    final long maxDiskOperationMillis = context.getProperty(MAX_DISK_OPERATION_TIME).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS);
    final long maxListingMillis = context.getProperty(MAX_LISTING_TIME).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS);

    final boolean trackPerformance = context.getProperty(TRACK_PERFORMANCE).asBoolean();
    if (trackPerformance) {
        final int maxEntries = context.getProperty(MAX_TRACKED_FILES).evaluateAttributeExpressions().asInteger();
        performanceTracker = new RollingMetricPerformanceTracker(getLogger(), maxDiskOperationMillis, maxEntries);
    } else {
        performanceTracker = new UntrackedPerformanceTracker(getLogger(), maxDiskOperationMillis);
    }

    final long millisToKeepStats = TimeUnit.MINUTES.toMillis(15);
    final MonitorActiveTasks monitorTask = new MonitorActiveTasks(performanceTracker, getLogger(), maxDiskOperationMillis, maxListingMillis, millisToKeepStats);
    monitoringFuture = monitoringThreadPool.scheduleAtFixedRate(monitorTask, 15, 15, TimeUnit.SECONDS);
}
 

@Override
protected ChannelDispatcher createDispatcher(final ProcessContext context, final BlockingQueue<LumberjackEvent> events) throws IOException {
    final EventFactory<LumberjackEvent> eventFactory = new LumberjackEventFactory();
    final ChannelHandlerFactory<LumberjackEvent, AsyncChannelDispatcher> handlerFactory = new LumberjackSocketChannelHandlerFactory<>();

    final int maxConnections = context.getProperty(MAX_CONNECTIONS).asInteger();
    final int bufferSize = context.getProperty(RECV_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
    final Charset charSet = Charset.forName(context.getProperty(CHARSET).getValue());

    // initialize the buffer pool based on max number of connections and the buffer size
    final BlockingQueue<ByteBuffer> bufferPool = createBufferPool(maxConnections, bufferSize);

    // if an SSLContextService was provided then create an SSLContext to pass down to the dispatcher
    SSLContext sslContext = null;
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    if (sslContextService != null) {
        sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.REQUIRED);
    }

    // if we decide to support SSL then get the context and pass it in here
    return new SocketChannelDispatcher<>(eventFactory, handlerFactory, bufferPool, events,
        getLogger(), maxConnections, sslContext, charSet);
}
 

@OnScheduled
public void OnScheduled(final ProcessContext context) {

    // Configure jackson mapper before spawning onTriggers
    final SimpleModule module = new SimpleModule()
                                    .addSerializer(MacAddress.class, new MacAddressToStringSerializer());
    mapper.registerModule(module);
    mapper.setDateFormat(this.simpleDateFormat);

    switch (context.getProperty(TIME_REPRESENTATION).getValue()) {
        case LOCAL_TZ:
            // set the mapper TZ to local TZ
            mapper.setTimeZone(TimeZone.getDefault());
            tzId = TimeZone.getDefault().getID();
            break;
        case UTC:
            // set the mapper TZ to local TZ
            mapper.setTimeZone(TimeZone.getTimeZone(UTC));
            tzId = UTC;
            break;
    }

}
 

@OnScheduled
public void onScheduled(final ProcessContext context) {
    final ComponentLog logger = getLogger();
    final Integer cacheSize = context.getProperty(CACHE_SIZE).asInteger();
    final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS);

    if (cacheSize > 0) {
        CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
        if (cacheTTL > 0) {
            cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS);
        }

        cache = cacheBuilder.build(
                new CacheLoader<String, Templates>() {
                    @Override
                    public Templates load(String path) throws TransformerConfigurationException, LookupFailureException {
                        return newTemplates(context, path);
                    }
                });
    } else {
        cache = null;
        logger.info("Stylesheet cache disabled because cache size is set to 0");
    }
}
 

/**
 * Will construct AMQP message by extracting its body from the incoming
 * {@link FlowFile}. AMQP Properties will be extracted from the
 * {@link FlowFile} and converted to {@link BasicProperties} to be sent
 * along with the message. Upon success the incoming {@link FlowFile} is
 * transferred to 'success' {@link Relationship} and upon failure FlowFile is
 * penalized and transferred to the 'failure' {@link Relationship}
 * <br>
 * NOTE: Attributes extracted from {@link FlowFile} are considered
 * candidates for AMQP properties if their names are prefixed with
 * {@link AMQPUtils#AMQP_PROP_PREFIX} (e.g., amqp$contentType=text/xml)
 *
 */
@Override
protected void rendezvousWithAmqp(ProcessContext context, ProcessSession processSession) throws ProcessException {
    FlowFile flowFile = processSession.get();
    if (flowFile != null) {
        BasicProperties amqpProperties = this.extractAmqpPropertiesFromFlowFile(flowFile);
        String routingKey = context.getProperty(ROUTING_KEY).evaluateAttributeExpressions(flowFile).getValue();
        if (routingKey == null){
            throw new IllegalArgumentException("Failed to determine 'routing key' with provided value '"
                    + context.getProperty(ROUTING_KEY) + "' after evaluating it as expression against incoming FlowFile.");
        }
        String exchange = context.getProperty(EXCHANGE).evaluateAttributeExpressions(flowFile).getValue();

        byte[] messageContent = this.extractMessage(flowFile, processSession);

        try {
            this.targetResource.publish(messageContent, amqpProperties, routingKey, exchange);
            processSession.transfer(flowFile, REL_SUCCESS);
            processSession.getProvenanceReporter().send(flowFile, this.amqpConnection.toString() + "/E:" + exchange + "/RK:" + routingKey);
        } catch (Exception e) {
            processSession.transfer(processSession.penalize(flowFile), REL_FAILURE);
            this.getLogger().error("Failed while sending message to AMQP via " + this.targetResource, e);
            context.yield();
        }
    }
}
 

/**
 * Method used to configure the 'sasl.jaas.config' property based on KAFKA-4259<br />
 * https://cwiki.apache.org/confluence/display/KAFKA/KIP-85%3A+Dynamic+JAAS+configuration+for+Kafka+clients<br />
 * <br />
 * It expects something with the following format: <br />
 * <br />
 * &lt;LoginModuleClass&gt; &lt;ControlFlag&gt; *(&lt;OptionName&gt;=&lt;OptionValue&gt;); <br />
 * ControlFlag = required / requisite / sufficient / optional
 *
 * @param mapToPopulate Map of configuration properties
 * @param context Context
 */
private static void setJaasConfig(Map<String, Object> mapToPopulate, ProcessContext context) {
    String keytab = context.getProperty(USER_KEYTAB).evaluateAttributeExpressions().getValue();
    String principal = context.getProperty(USER_PRINCIPAL).evaluateAttributeExpressions().getValue();

    // If the Kerberos Credentials Service is specified, we need to use its configuration, not the explicit properties for principal/keytab.
    // The customValidate method ensures that only one can be set, so we know that the principal & keytab above are null.
    final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
    if (credentialsService != null) {
        principal = credentialsService.getPrincipal();
        keytab = credentialsService.getKeytab();
    }


    String serviceName = context.getProperty(JAAS_SERVICE_NAME).evaluateAttributeExpressions().getValue();
    if(StringUtils.isNotBlank(keytab) && StringUtils.isNotBlank(principal) && StringUtils.isNotBlank(serviceName)) {
        mapToPopulate.put(SaslConfigs.SASL_JAAS_CONFIG, "com.sun.security.auth.module.Krb5LoginModule required "
                + "useTicketCache=false "
                + "renewTicket=true "
                + "serviceName=\"" + serviceName + "\" "
                + "useKeyTab=true "
                + "keyTab=\"" + keytab + "\" "
                + "principal=\"" + principal + "\";");
    }
}
 

protected AWSCredentials getCredentials(final ProcessContext context) {
    final String accessKey = context.getProperty(ACCESS_KEY).evaluateAttributeExpressions().getValue();
    final String secretKey = context.getProperty(SECRET_KEY).evaluateAttributeExpressions().getValue();

    final String credentialsFile = context.getProperty(CREDENTIALS_FILE).getValue();

    if (credentialsFile != null) {
        try {
            return new PropertiesCredentials(new File(credentialsFile));
        } catch (final IOException ioe) {
            throw new ProcessException("Could not read Credentials File", ioe);
        }
    }

    if (accessKey != null && secretKey != null) {
        return new BasicAWSCredentials(accessKey, secretKey);
    }

    return new AnonymousAWSCredentials();

}
 

protected void initializeRegionAndEndpoint(ProcessContext context) {
    // if the processor supports REGION, get the configured region.
    if (getSupportedPropertyDescriptors().contains(REGION)) {
        final String region = context.getProperty(REGION).getValue();
        if (region != null) {
            this.region = Region.getRegion(Regions.fromName(region));
            client.setRegion(this.region);
        } else {
            this.region = null;
        }
    }

    // if the endpoint override has been configured, set the endpoint.
    // (per Amazon docs this should only be configured at client creation)
    final String urlstr = StringUtils.trimToEmpty(context.getProperty(ENDPOINT_OVERRIDE).getValue());
    if (!urlstr.isEmpty()) {
        this.client.setEndpoint(urlstr);
    }

}
 

/**
 * Register subscriber via native call
 *
 * @param context the process context
 */
private String subscribe(ProcessContext context) throws URISyntaxException {
    String channel = context.getProperty(CHANNEL).getValue();
    String query = context.getProperty(QUERY).getValue();

    renderedXMLs = new LinkedBlockingQueue<>(context.getProperty(MAX_EVENT_QUEUE_SIZE).asInteger());
    provenanceUri = new URI("winlog", name, "/" + channel, query, null).toASCIIString();

    evtSubscribeCallback = new EventSubscribeXmlRenderingCallback(getLogger(), s -> {
        try {
            renderedXMLs.put(s);
        } catch (InterruptedException e) {
            throw new IllegalStateException("Got interrupted while waiting to add to queue.", e);
        }
    }, context.getProperty(MAX_BUFFER_SIZE).asInteger(), wEvtApi, kernel32, errorLookup);

    subscriptionHandle = wEvtApi.EvtSubscribe(null, null, channel, query, null, null,
            evtSubscribeCallback, WEvtApi.EvtSubscribeFlags.SUBSCRIBE_TO_FUTURE | WEvtApi.EvtSubscribeFlags.EVT_SUBSCRIBE_STRICT);
    if (!isSubscribed()) {
        return UNABLE_TO_SUBSCRIBE + errorLookup.getLastError();
    }
    return null;
}
 

/**
 * Adapted from HBASEUtils. Their approach seemed ideal for what our intent is here.
 * @param columnFamily column family from which to extract the visibility or to execute an expression against
 * @param columnQualifier column qualifier from which to extract the visibility or to execute an expression against
 * @param flowFile flow file being written
 * @param context process context
 * @return
 */
public static String produceVisibility(String columnFamily, String columnQualifier, FlowFile flowFile, ProcessContext context) {
    if (org.apache.commons.lang3.StringUtils.isNotEmpty(columnFamily)) {
        return null;
    }
    String lookupKey = String.format("visibility.%s%s%s", columnFamily, !org.apache.commons.lang3.StringUtils.isNotEmpty(columnQualifier) ? "." : "", columnQualifier);
    String fromAttribute = flowFile.getAttribute(lookupKey);

    if (fromAttribute == null && !org.apache.commons.lang3.StringUtils.isBlank(columnQualifier)) {
        String lookupKeyFam = String.format("visibility.%s", columnFamily);
        fromAttribute = flowFile.getAttribute(lookupKeyFam);
    }

    if (fromAttribute != null) {
        return fromAttribute;
    } else {
        PropertyValue descriptor = context.getProperty(lookupKey);
        if (descriptor == null || !descriptor.isSet()) {
            descriptor = context.getProperty(String.format("visibility.%s", columnFamily));
        }

        String retVal = descriptor != null ? descriptor.evaluateAttributeExpressions(flowFile).getValue() : null;

        return retVal;
    }
}
 

public RegexReplace(final byte[] buffer, final ProcessContext context) {
    this.buffer = buffer;

    final String regexValue = context.getProperty(SEARCH_VALUE).evaluateAttributeExpressions().getValue();
    numCapturingGroups = Pattern.compile(regexValue).matcher("").groupCount();
    additionalAttrs = new HashMap<>(numCapturingGroups);
}
 

/**
 * Will builds target resource ({@link AMQPPublisher} or
 * {@link AMQPConsumer}) upon first invocation and will delegate to the
 * implementation of
 * {@link #rendezvousWithAmqp(ProcessContext, ProcessSession)} method for
 * further processing.
 */
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    synchronized (this) {
        this.buildTargetResource(context);
    }
    this.rendezvousWithAmqp(context, session);
}
 

/**
 * Send the entire FlowFile as a single message.
 */
private void processSingleMessage(ProcessContext context, ProcessSession session, FlowFile flowFile, ChannelSender sender) {
    // copy the contents of the FlowFile to the ByteArrayOutputStream
    final ByteArrayOutputStream baos = new ByteArrayOutputStream((int)flowFile.getSize() + 1);
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.copy(in, baos);
        }
    });

    // if TCP and we don't end in a new line then add one
    final String protocol = context.getProperty(PROTOCOL).getValue();
    byte[] buf = baos.toByteArray();
    if (protocol.equals(TCP_VALUE.getValue()) && buf[buf.length - 1] != NEW_LINE_CHAR) {
        final byte[] updatedBuf = new byte[buf.length + 1];
        System.arraycopy(buf, 0, updatedBuf, 0, buf.length);
        updatedBuf[updatedBuf.length - 1] = NEW_LINE_CHAR;
        buf = updatedBuf;
    }

    // create a message batch of one message and add to active batches
    final FlowFileMessageBatch messageBatch = new FlowFileMessageBatch(session, flowFile);
    messageBatch.setNumMessages(1);
    activeBatches.add(messageBatch);

    // attempt to send the data and add the appropriate range
    try {
        sender.send(buf);
        messageBatch.addSuccessfulRange(0L, flowFile.getSize());
    } catch (IOException e) {
        messageBatch.addFailedRange(0L, flowFile.getSize(), e);
        context.yield();
    }
}
 

PartitionReceiver getReceiver(final ProcessContext context, final String partitionId) throws IOException, EventHubException, ExecutionException, InterruptedException {
    PartitionReceiver existingReceiver = partitionToReceiverMap.get(partitionId);
    if (existingReceiver != null) {
        return existingReceiver;
    }

    // we want to avoid allowing multiple threads to create Receivers simultaneously because that could result in
    // having multiple Receivers for the same partition. So if the map does not contain a receiver for this partition,
    // we will enter a synchronized block and check again (because once we enter the synchronized block, we know that no
    // other thread is creating a client). If within the synchronized block, we still do not have an entry in the map,
    // it is up to use to create the receiver, initialize it, and then put it into the map.
    // We do not use the putIfAbsent method in order to do a CAS operation here because we want to also initialize the
    // receiver if and only if it is not present in the map. As a result, we need to initialize the receiver and add it
    // to the map atomically. Hence, the synchronized block.
    synchronized (this) {
        existingReceiver = partitionToReceiverMap.get(partitionId);
        if (existingReceiver != null) {
            return existingReceiver;
        }

        final String consumerGroupName = context.getProperty(CONSUMER_GROUP).getValue();

        final PartitionReceiver receiver = eventHubClient.createReceiver(
                consumerGroupName,
                partitionId,
                EventPosition.fromEnqueuedTime(
                        configuredEnqueueTime == null ? Instant.now() : configuredEnqueueTime)).get();

        receiver.setReceiveTimeout(receiverFetchTimeout == null ? Duration.ofMillis(60000) : receiverFetchTimeout);
        partitionToReceiverMap.put(partitionId, receiver);
        return receiver;

    }
}
 

private byte[] getDelimiterTextContent(final ProcessContext context, final List<FlowFile> flowFiles, final PropertyDescriptor descriptor)
        throws IOException {
    byte[] property = null;
    if (flowFiles != null && flowFiles.size() > 0) {
        final FlowFile flowFile = flowFiles.get(0);
        if (flowFile != null) {
            final String value = context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue();
            if (value != null) {
                property = value.getBytes(StandardCharsets.UTF_8);
            }
        }
    }
    return property;
}
 

/**
 * Create CannedAccessControlList if {@link #CANNED_ACL} property specified.
 *
 * @param context ProcessContext
 * @param flowFile FlowFile
 * @return CannedAccessControlList or null if not specified
 */
protected final CannedAccessControlList createCannedACL(final ProcessContext context, final FlowFile flowFile) {
    CannedAccessControlList cannedAcl = null;

    final String cannedAclString = context.getProperty(CANNED_ACL).evaluateAttributeExpressions(flowFile).getValue();
    if (!StringUtils.isEmpty(cannedAclString)) {
        cannedAcl = CannedAccessControlList.valueOf(cannedAclString);
    }

    return cannedAcl;
}
 

/**
 * Create client using aws credentials provider. This is the preferred way for creating clients
 */
@Override
protected AmazonKinesisFirehoseClient createClient(final ProcessContext context, final AWSCredentialsProvider credentialsProvider, final ClientConfiguration config) {
    getLogger().info("Creating client using aws credentials provider");

    return new AmazonKinesisFirehoseClient(credentialsProvider, config);
}
 

InvocationContextProperties(ProcessContext context, FlowFile flowFile) {
    List<PropertyDescriptor> propertyDescriptors = BaseTransformer.this.getSupportedPropertyDescriptors();
    for (PropertyDescriptor propertyDescriptor : propertyDescriptors) {
        if (propertyDescriptor.isExpressionLanguageSupported()) {
            PropertyValue value = context.getProperty(propertyDescriptor)
                    .evaluateAttributeExpressions(flowFile);
            this.propertyInvocationValues.put(propertyDescriptor, value.getValue());
        }
    }
}
 

private boolean evaluateRule(final ProcessContext context, final Rule rule, FlowFile flowfile, final Map<String, String> statefulAttributes) {
    // go through each condition
    for (final Condition condition : rule.getConditions()) {

        // fail if any condition is not met
        if (!evaluateCondition(context, condition, flowfile, statefulAttributes)) {
            return false;
        }
    }

    return true;
}
 

@Override
protected ChannelDispatcher createDispatcher(final ProcessContext context, final BlockingQueue<BeatsEvent> events) throws IOException {
    final EventFactory<BeatsEvent> eventFactory = new BeatsEventFactory();
    final ChannelHandlerFactory<BeatsEvent, AsyncChannelDispatcher> handlerFactory = new BeatsSocketChannelHandlerFactory<>();

    final int maxConnections = context.getProperty(MAX_CONNECTIONS).asInteger();
    final int bufferSize = context.getProperty(RECV_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
    final Charset charSet = Charset.forName(context.getProperty(CHARSET).getValue());

    // initialize the buffer pool based on max number of connections and the buffer size
    final BlockingQueue<ByteBuffer> bufferPool = createBufferPool(maxConnections, bufferSize);

    // if an SSLContextService was provided then create an SSLContext to pass down to the dispatcher
    SSLContext sslContext = null;
    SslContextFactory.ClientAuth clientAuth = null;
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    if (sslContextService != null) {
        final String clientAuthValue = context.getProperty(CLIENT_AUTH).getValue();
        sslContext = sslContextService.createSSLContext(SslContextFactory.ClientAuth.valueOf(clientAuthValue));
        clientAuth = SslContextFactory.ClientAuth.valueOf(clientAuthValue);

    }

    // if we decide to support SSL then get the context and pass it in here
    return new SocketChannelDispatcher<>(eventFactory, handlerFactory, bufferPool, events,
        getLogger(), maxConnections, sslContext, clientAuth, charSet);
}
 

/**
 * Add failed flow file attributes
 * @param flowFiles all flow files
 * @param failedFlowFiles list of failed flow files
 * @param session process session
 * @param context the process context
 * @return failed flow files with updated attributes
 */
protected List<FlowFile> updateFailedFlowFileAttributes(
        List<FlowFile> flowFiles,
        List<FlowFile> failedFlowFiles, ProcessSession session, ProcessContext context) {

    int flowFileCount = flowFiles.size();
    int flowFileFailed = failedFlowFiles.size();
    List<FlowFile> updatedFailedFlowFiles = new ArrayList<>();

    for (int i = 0; i < flowFileFailed; i++) {
        FlowFile flowFile = failedFlowFiles.get(i);

        Map<String,String> attributes = new HashMap<>();
        attributes.put(IGNITE_BATCH_FLOW_FILE_FAILED_ITEM_NUMBER, Integer.toString(i));
        attributes.put(IGNITE_BATCH_FLOW_FILE_TOTAL_COUNT, Integer.toString(flowFileCount));
        attributes.put(IGNITE_BATCH_FLOW_FILE_ITEM_NUMBER, Integer.toString(flowFiles.indexOf(flowFile)));
        attributes.put(IGNITE_BATCH_FLOW_FILE_FAILED_COUNT, Integer.toString(flowFileFailed));

        String key = context.getProperty(IGNITE_CACHE_ENTRY_KEY).evaluateAttributeExpressions(flowFile).getValue();

        if (StringUtils.isEmpty(key)) {
            attributes.put(IGNITE_BATCH_FLOW_FILE_FAILED_REASON_ATTRIBUTE_KEY,
                    IGNITE_BATCH_FLOW_FILE_FAILED_MISSING_KEY_MESSAGE);
        } else if (flowFile.getSize() == 0) {
            attributes.put(IGNITE_BATCH_FLOW_FILE_FAILED_REASON_ATTRIBUTE_KEY,
                    IGNITE_BATCH_FLOW_FILE_FAILED_ZERO_SIZE_MESSAGE);
        } else {
            throw new ProcessException("Unknown reason for failing file: " + flowFile);
        }

        flowFile = session.putAllAttributes(flowFile, attributes);
        updatedFailedFlowFiles.add(flowFile);
    }

    return updatedFailedFlowFiles;

}
 

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    final FlowFile original = session.get();
    if (original == null) {
        return;
    }

    final Map<String, String> atrList = buildAttributesMapForFlowFile(original, attributes, attributesToRemove, nullValueForEmptyString, pattern);

    try {
        if (destinationContent) {
            FlowFile conFlowfile = session.write(original, (in, out) -> {
                try (OutputStream outputStream = new BufferedOutputStream(out)) {
                    outputStream.write(objectMapper.writeValueAsBytes(atrList));
                }
            });
            conFlowfile = session.putAttribute(conFlowfile, CoreAttributes.MIME_TYPE.key(), APPLICATION_JSON);
            session.transfer(conFlowfile, REL_SUCCESS);
        } else {
            FlowFile atFlowfile = session.putAttribute(original, JSON_ATTRIBUTE_NAME, objectMapper.writeValueAsString(atrList));
            session.transfer(atFlowfile, REL_SUCCESS);
        }
    } catch (JsonProcessingException e) {
        getLogger().error(e.getMessage());
        session.transfer(original, REL_FAILURE);
    }
}
 

@OnScheduled
public void setup(ProcessContext context) {
    // If the query is not set, then an incoming flow file is needed. Otherwise fail the initialization
    if (!context.getProperty(HIVEQL_SELECT_QUERY).isSet() && !context.hasIncomingConnection()) {
        final String errorString = "Either the Select Query must be specified or there must be an incoming connection "
                + "providing flowfile(s) containing a SQL select query";
        getLogger().error(errorString);
        throw new ProcessException(errorString);
    }
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile input = session.get();
    if (input == null) {
        return;
    }

    final String deleteQuery = getQuery(context, input);
    final String queryAttribute = context.getProperty(QUERY_ATTRIBUTE).isSet()
            ? context.getProperty(QUERY_ATTRIBUTE).evaluateAttributeExpressions(input).getValue()
            : null;
    GridFSBucket bucket = getBucket(input, context);

    try {
        Document query = Document.parse(deleteQuery);
        MongoCursor cursor = bucket.find(query).iterator();
        if (cursor.hasNext()) {
            GridFSFile file = (GridFSFile)cursor.next();
            bucket.delete(file.getObjectId());

            if (!StringUtils.isEmpty(queryAttribute)) {
                input = session.putAttribute(input, queryAttribute, deleteQuery);
            }

            session.transfer(input, REL_SUCCESS);
        } else {
            getLogger().error(String.format("Query %s did not delete anything in %s", deleteQuery, bucket.getBucketName()));
            session.transfer(input, REL_FAILURE);
        }

        cursor.close();
    } catch (Exception ex) {
        getLogger().error(String.format("Error deleting using query: %s", deleteQuery), ex);
        session.transfer(input, REL_FAILURE);
    }
}
 

@Test
public void testInvalid() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    final int numMessages = 10;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // write some TCP messages to the port in the background
    final Thread sender = new Thread(new SingleConnectionSocketSender(port, numMessages, 100, INVALID_MESSAGE));
    sender.setDaemon(true);
    sender.start();

    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;

        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(50);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_INVALID).size();
        }

        // all messages should be transferred to invalid
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);

    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}
 

public static Connection createConnection(final ProcessContext context, final String clientId) throws JMSException {
    Objects.requireNonNull(context);
    Objects.requireNonNull(clientId);

    final ConnectionFactory connectionFactory = createConnectionFactory(context);

    final String username = context.getProperty(USERNAME).getValue();
    final String password = context.getProperty(PASSWORD).getValue();
    final Connection connection = (username == null && password == null) ? connectionFactory.createConnection() : connectionFactory.createConnection(username, password);

    connection.setClientID(clientId);
    connection.start();
    return connection;
}
 

/**
 *
 */
protected FlowFile doTransform(ProcessContext context, ProcessSession session, FlowFile flowFile,  InvocationContextProperties contextProperties) {
    AtomicReference<Map<String, String>> attributeRef = new AtomicReference<Map<String, String>>();
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(InputStream in) throws IOException {
            attributeRef.set(transform(in, null, contextProperties));
        }
    });
    if (attributeRef.get() != null) {
        flowFile = session.putAllAttributes(flowFile, attributeRef.get());
    }
    return flowFile;
}
 

private Publisher.Builder getPublisherBuilder(ProcessContext context) {
    final Long batchSize = context.getProperty(BATCH_SIZE).asLong();

    return Publisher.newBuilder(getTopicName(context))
            .setCredentialsProvider(FixedCredentialsProvider.create(getGoogleCredentials(context)))
            .setBatchingSettings(BatchingSettings.newBuilder()
            .setElementCountThreshold(batchSize)
            .setIsEnabled(true)
            .build());
}