Java Code Examples for org.apache.nifi.logging.ComponentLog#info()

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final List<FlowFile> flowFiles = session.get(50);
    if (flowFiles.isEmpty()) {
        return;
    }

    final ComponentLog logger = getLogger();
    try {
        if (fileWatcher.checkAndReset()) {
            this.dictionaryTerms = createDictionary(context);
        }
    } catch (final IOException e) {
        logger.error("Unable to reload dictionary due to {}", e);
    }

    final boolean matchAll = context.getProperty(MATCHING_CRITERIA).getValue().equals(MATCH_CRITERIA_ALL);

    for (final FlowFile flowFile : flowFiles) {
        final boolean matched = matchAll ? allMatch(flowFile, attributePattern, dictionaryTerms) : anyMatch(flowFile, attributePattern, dictionaryTerms);
        final Relationship relationship = matched ? REL_MATCHED : REL_UNMATCHED;
        session.getProvenanceReporter().route(flowFile, relationship);
        session.transfer(flowFile, relationship);
        logger.info("Transferred {} to {}", new Object[]{flowFile, relationship});
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final Map<String, Pattern> patterns = regexMapRef.get();
    final ComponentLog logger = getLogger();

    final SortedMap<String, String> attributes = getRelevantAttributes(flowFile, patterns);
    if (attributes.size() != patterns.size()) {
        final Set<String> wantedKeys = patterns.keySet();
        final Set<String> foundKeys = attributes.keySet();
        final StringBuilder missingKeys = new StringBuilder();
        for (final String wantedKey : wantedKeys) {
            if (!foundKeys.contains(wantedKey)) {
                missingKeys.append(wantedKey).append(" ");
            }
        }

        logger.error("routing {} to 'failure' because of missing attributes: {}", new Object[]{flowFile, missingKeys.toString()});
        session.transfer(flowFile, REL_FAILURE);
    } else {
        // create single string of attribute key/value pairs to use for group ID hash
        final StringBuilder hashableValue = new StringBuilder();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            hashableValue.append(entry.getKey());
            if (StringUtils.isBlank(entry.getValue())) {
                hashableValue.append("EMPTY");
            } else {
                hashableValue.append(entry.getValue());
            }
        }

        // create group ID
        final String hashValue = DigestUtils.md5Hex(hashableValue.toString());

        logger.info("adding Hash Value {} to attributes for {} and routing to success", new Object[]{hashValue, flowFile});
        flowFile = session.putAttribute(flowFile, context.getProperty(HASH_VALUE_ATTRIBUTE).getValue(), hashValue);
        session.getProvenanceReporter().modifyAttributes(flowFile);
        session.transfer(flowFile, REL_SUCCESS);
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final List<FlowFile> flowFiles = session.get(50);
    if (flowFiles.isEmpty()) {
        return;
    }

    final Schema schema = schemaRef.get();
    final Validator validator = schema.newValidator();
    final ComponentLog logger = getLogger();

    for (final FlowFile flowFile : flowFiles) {
        final AtomicBoolean valid = new AtomicBoolean(true);
        session.read(flowFile, new InputStreamCallback() {
            @Override
            public void process(final InputStream in) throws IOException {
                try {
                    validator.validate(new StreamSource(in));
                } catch (final IllegalArgumentException | SAXException e) {
                    valid.set(false);
                    logger.debug("Failed to validate {} against schema due to {}", new Object[]{flowFile, e});
                }
            }
        });

        if (valid.get()) {
            logger.info("Successfully validated {} against schema; routing to 'valid'", new Object[]{flowFile});
            session.getProvenanceReporter().route(flowFile, REL_VALID);
            session.transfer(flowFile, REL_VALID);
        } else {
            logger.info("Failed to validate {} against schema; routing to 'invalid'", new Object[]{flowFile});
            session.getProvenanceReporter().route(flowFile, REL_INVALID);
            session.transfer(flowFile, REL_INVALID);
        }
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    updateMapping(context);
    final List<FlowFile> flowFiles = session.get(5);
    if (flowFiles.isEmpty()) {
        return;
    }

    final ComponentLog logger = getLogger();

    final int maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();

    for (FlowFile flowFile : flowFiles) {
        if (flowFile.getSize() > maxBufferSize) {
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }

        final StopWatch stopWatch = new StopWatch(true);

        flowFile = session.write(flowFile, new ReplaceTextCallback(context, flowFile, maxBufferSize));

        logger.info("Transferred {} to 'success'", new Object[]{flowFile});
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    }
}
 

private void processFlowFile(
        final ComponentLog logger,
        final MessageLogLevel logLevel,
        final FlowFile flowFile,
        final ProcessContext context) {

    String logPrefix = context.getProperty(LOG_PREFIX).evaluateAttributeExpressions(flowFile).getValue();
    String logMessage = context.getProperty(LOG_MESSAGE).evaluateAttributeExpressions(flowFile).getValue();

    String messageToWrite;
    if (StringUtil.isBlank(logPrefix)) {
        messageToWrite = logMessage;
    } else {
        messageToWrite = String.format("%s%s", logPrefix, logMessage);
    }

    // Uses optional property to specify logging level
    switch (logLevel) {
        case info:
            logger.info(messageToWrite);
            break;
        case debug:
            logger.debug(messageToWrite);
            break;
        case warn:
            logger.warn(messageToWrite);
            break;
        case trace:
            logger.trace(messageToWrite);
            break;
        case error:
            logger.error(messageToWrite);
            break;
        default:
            logger.debug(messageToWrite);
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final Charset charset = Charset.forName(context.getProperty(CHARACTER_SET).getValue());
    final int maxCaptureGroupLength = context.getProperty(MAX_CAPTURE_GROUP_LENGTH).asInteger();

    final String contentString;
    byte[] buffer = bufferQueue.poll();
    if (buffer == null) {
        final int maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
        buffer = new byte[maxBufferSize];
    }

    try {
        final byte[] byteBuffer = buffer;
        session.read(flowFile, new InputStreamCallback() {
            @Override
            public void process(InputStream in) throws IOException {
                StreamUtils.fillBuffer(in, byteBuffer, false);
            }
        });

        final long len = Math.min(byteBuffer.length, flowFile.getSize());
        contentString = new String(byteBuffer, 0, (int) len, charset);
    } finally {
        bufferQueue.offer(buffer);
    }

    final Map<String, String> regexResults = new HashMap<>();

    final Map<String, Pattern> patternMap = compiledPattersMapRef.get();

    final int startGroupIdx = context.getProperty(INCLUDE_CAPTURE_GROUP_ZERO).asBoolean() ? 0 : 1;

    for (final Map.Entry<String, Pattern> entry : patternMap.entrySet()) {

        final Matcher matcher = entry.getValue().matcher(contentString);
        int j = 0;

        while (matcher.find()) {
            final String baseKey = entry.getKey();
            int start = j == 0 ? startGroupIdx : 1;
            for (int i = start; i <= matcher.groupCount(); i++) {
                final String key = new StringBuilder(baseKey).append(".").append(i + j).toString();
                String value = matcher.group(i);
                if (value != null && !value.isEmpty()) {
                    if (value.length() > maxCaptureGroupLength) {
                        value = value.substring(0, maxCaptureGroupLength);
                    }
                    regexResults.put(key, value);
                    if (i == 1 && j == 0) {
                        regexResults.put(baseKey, value);
                    }
                }
            }
            j += matcher.groupCount();
            if (!context.getProperty(ENABLE_REPEATING_CAPTURE_GROUP).asBoolean()) {
                break;
            }
        }
    }

    if (!regexResults.isEmpty()) {
        flowFile = session.putAllAttributes(flowFile, regexResults);
        session.getProvenanceReporter().modifyAttributes(flowFile);
        session.transfer(flowFile, REL_MATCH);
        logger.info("Matched {} Regular Expressions and added attributes to FlowFile {}", new Object[]{regexResults.size(), flowFile});
    } else {
        session.transfer(flowFile, REL_NO_MATCH);
        logger.info("Did not match any Regular Expressions for  FlowFile {}", new Object[]{flowFile});
    }

}
 

@Override
protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) {
    final Configuration conf = getConfiguration();
    final FileSystem hdfs = getFileSystem();
    final String flowFileContentValue = context.getProperty(FLOWFILE_CONTENT).getValue();
    final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean();
    final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B);
    if (bufferSizeProp != null) {
        int bufferSize = bufferSizeProp.intValue();
        conf.setInt(BUFFER_SIZE_KEY, bufferSize);
    }
    ComponentLog logger = getLogger();
    final SequenceFileReader<Set<FlowFile>> reader;
    if (flowFileContentValue.equalsIgnoreCase(VALUE_ONLY)) {
        reader = new ValueReader(session);
    } else {
        reader = new KeyValueReader(session);
    }
    Set<FlowFile> flowFiles = Collections.emptySet();
    for (final Path file : files) {
        if (!this.isScheduled()) {
            break; // This processor should stop running immediately.
        }

        final StopWatch stopWatch = new StopWatch(false);
        try {
            stopWatch.start();
            if (!hdfs.exists(file)) {
                continue; // If file is no longer here move on.
            }
            logger.debug("Reading file");
            flowFiles = getFlowFiles(conf, hdfs, reader, file);
            if (!keepSourceFiles && !hdfs.delete(file, false)) {
                logger.warn("Unable to delete path " + file.toString() + " from HDFS.  Will likely be picked up over and over...");
            }
        } catch (Throwable t) {
            logger.error("Error retrieving file {} from HDFS due to {}", new Object[]{file, t});
            session.rollback();
            context.yield();
        } finally {
            stopWatch.stop();
            long totalSize = 0;
            for (FlowFile flowFile : flowFiles) {
                totalSize += flowFile.getSize();
                session.getProvenanceReporter().receive(flowFile, file.toString());
            }
            if (totalSize > 0) {
                final String dataRate = stopWatch.calculateDataRate(totalSize);
                final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
                logger.info("Created {} flowFiles from SequenceFile {}. Ingested in {} milliseconds at a rate of {}", new Object[]{
                    flowFiles.size(), file.toUri().toASCIIString(), millis, dataRate});
                logger.info("Transferred flowFiles {}  to success", new Object[]{flowFiles});
                session.transfer(flowFiles, REL_SUCCESS);
            }
        }
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();
    final String cacheKey = context.getProperty(CACHE_ENTRY_IDENTIFIER).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isBlank(cacheKey)) {
        logger.error("FlowFile {} has no attribute for given Cache Entry Identifier", new Object[]{flowFile});
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final DistributedMapCacheClient cache = context.getProperty(DISTRIBUTED_CACHE_SERVICE).asControllerService(DistributedMapCacheClient.class);
    final Long durationMS = context.getProperty(AGE_OFF_DURATION).asTimePeriod(TimeUnit.MILLISECONDS);
    final long now = System.currentTimeMillis();

    try {
        final String flowFileDescription = context.getProperty(FLOWFILE_DESCRIPTION).evaluateAttributeExpressions(flowFile).getValue();
        final CacheValue cacheValue = new CacheValue(flowFileDescription, now);
        final CacheValue originalCacheValue;

        final boolean shouldCacheIdentifier = context.getProperty(CACHE_IDENTIFIER).asBoolean();
        if (shouldCacheIdentifier) {
            originalCacheValue = cache.getAndPutIfAbsent(cacheKey, cacheValue, keySerializer, valueSerializer, valueDeserializer);
        } else {
            originalCacheValue = cache.get(cacheKey, keySerializer, valueDeserializer);
        }

        boolean duplicate = originalCacheValue != null;
        if (duplicate && durationMS != null && (now >= originalCacheValue.getEntryTimeMS() + durationMS)) {
            boolean status = cache.remove(cacheKey, keySerializer);
            logger.debug("Removal of expired cached entry with key {} returned {}", new Object[]{cacheKey, status});

            // both should typically result in duplicate being false...but, better safe than sorry
            if (shouldCacheIdentifier) {
                duplicate = !cache.putIfAbsent(cacheKey, cacheValue, keySerializer, valueSerializer);
            } else {
                duplicate = cache.containsKey(cacheKey, keySerializer);
            }
        }

        if (duplicate) {
            session.getProvenanceReporter().route(flowFile, REL_DUPLICATE, "Duplicate of: " + ORIGINAL_DESCRIPTION_ATTRIBUTE_NAME);
            String originalFlowFileDescription = originalCacheValue.getDescription();
            flowFile = session.putAttribute(flowFile, ORIGINAL_DESCRIPTION_ATTRIBUTE_NAME, originalFlowFileDescription);
            session.transfer(flowFile, REL_DUPLICATE);
            logger.info("Found {} to be a duplicate of FlowFile with description {}", new Object[]{flowFile, originalFlowFileDescription});
            session.adjustCounter("Duplicates Detected", 1L, false);
        } else {
            session.getProvenanceReporter().route(flowFile, REL_NON_DUPLICATE);
            session.transfer(flowFile, REL_NON_DUPLICATE);
            logger.info("Could not find a duplicate entry in cache for {}; routing to non-duplicate", new Object[]{flowFile});
            session.adjustCounter("Non-Duplicate Files Processed", 1L, false);
        }
    } catch (final IOException e) {
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        logger.error("Unable to communicate with cache when processing {} due to {}", new Object[]{flowFile, e});
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final FlowFile original = session.get();
    if (original == null) {
        return;
    }

    final int depth = context.getProperty(SPLIT_DEPTH).asInteger();
    final ComponentLog logger = getLogger();

    final List<FlowFile> splits = new ArrayList<>();
    final String fragmentIdentifier = UUID.randomUUID().toString();
    final AtomicInteger numberOfRecords = new AtomicInteger(0);
    final XmlSplitterSaxParser parser = new XmlSplitterSaxParser(xmlTree -> {
        FlowFile split = session.create(original);
        split = session.write(split, out -> out.write(xmlTree.getBytes("UTF-8")));
        split = session.putAttribute(split, FRAGMENT_ID.key(), fragmentIdentifier);
        split = session.putAttribute(split, FRAGMENT_INDEX.key(), Integer.toString(numberOfRecords.getAndIncrement()));
        split = session.putAttribute(split, SEGMENT_ORIGINAL_FILENAME.key(), split.getAttribute(CoreAttributes.FILENAME.key()));
        splits.add(split);
    }, depth);

    final AtomicBoolean failed = new AtomicBoolean(false);
    session.read(original, rawIn -> {
        try (final InputStream in = new java.io.BufferedInputStream(rawIn)) {
            try {
                final XMLReader reader = XmlUtils.createSafeSaxReader(saxParserFactory, parser);
                reader.parse(new InputSource(in));
            } catch (final ParserConfigurationException | SAXException e) {
                logger.error("Unable to parse {} due to {}", new Object[]{original, e});
                failed.set(true);
            }
        }
    });

    if (failed.get()) {
        session.transfer(original, REL_FAILURE);
        session.remove(splits);
    } else {
        splits.forEach((split) -> {
            split = session.putAttribute(split, FRAGMENT_COUNT.key(), Integer.toString(numberOfRecords.get()));
            session.transfer(split, REL_SPLIT);
        });

        final FlowFile originalToTransfer = copyAttributesToOriginal(session, original, fragmentIdentifier, numberOfRecords.get());
        session.transfer(originalToTransfer, REL_ORIGINAL);
        logger.info("Split {} into {} FlowFiles", new Object[]{originalToTransfer, splits.size()});
    }
}
 

@OnScheduled
public void startServer(final ProcessContext context) throws NoSuchAlgorithmException, IOException, KeyStoreException, CertificateException, UnrecoverableKeyException {
    final ComponentLog logger = getLogger();
    // gather configured properties
    final Integer port = context.getProperty(PROP_SERVICE_PORT).asInteger();
    final Boolean useSecure = context.getProperty(PROP_USE_SECURE).asBoolean();
    final Integer flowControlWindow = context.getProperty(PROP_FLOW_CONTROL_WINDOW).asDataSize(DataUnit.B).intValue();
    final Integer maxMessageSize = context.getProperty(PROP_MAX_MESSAGE_SIZE).asDataSize(DataUnit.B).intValue();
    final SSLContextService sslContextService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    final SSLContext sslContext = sslContextService == null ? null : sslContextService.createSSLContext(SslContextFactory.ClientAuth.NONE);
    final Pattern authorizedDnPattern = Pattern.compile(context.getProperty(PROP_AUTHORIZED_DN_PATTERN).getValue());
    final FlowFileIngestServiceInterceptor callInterceptor = new FlowFileIngestServiceInterceptor(getLogger());
    callInterceptor.enforceDNPattern(authorizedDnPattern);

    final FlowFileIngestService flowFileIngestService = new FlowFileIngestService(getLogger(),
            sessionFactoryReference,
            context);
    NettyServerBuilder serverBuilder = NettyServerBuilder.forPort(port)
            .addService(ServerInterceptors.intercept(flowFileIngestService, callInterceptor))
            // default (de)compressor registries handle both plaintext and gzip compressed messages
            .compressorRegistry(CompressorRegistry.getDefaultInstance())
            .decompressorRegistry(DecompressorRegistry.getDefaultInstance())
            .flowControlWindow(flowControlWindow)
            .maxMessageSize(maxMessageSize);

    if (useSecure && sslContext != null) {
        // construct key manager
        if (StringUtils.isBlank(sslContextService.getKeyStoreFile())) {
            throw new IllegalStateException("SSL is enabled, but no keystore has been configured. You must configure a keystore.");
        }

        final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm(),
                sslContext.getProvider());
        final KeyStore keyStore = KeyStore.getInstance(sslContextService.getKeyStoreType());
        try (final InputStream is = new FileInputStream(sslContextService.getKeyStoreFile())) {
            keyStore.load(is, sslContextService.getKeyStorePassword().toCharArray());
        }
        keyManagerFactory.init(keyStore, sslContextService.getKeyStorePassword().toCharArray());

        SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(keyManagerFactory);

        // if the trust store is configured, then client auth is required.
        if (StringUtils.isNotBlank(sslContextService.getTrustStoreFile())) {
            final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm(),
                    sslContext.getProvider());
            final KeyStore trustStore = KeyStore.getInstance(sslContextService.getTrustStoreType());
            try (final InputStream is = new FileInputStream(sslContextService.getTrustStoreFile())) {
                trustStore.load(is, sslContextService.getTrustStorePassword().toCharArray());
            }
            trustManagerFactory.init(trustStore);
            sslContextBuilder = sslContextBuilder.trustManager(trustManagerFactory);
            sslContextBuilder = sslContextBuilder.clientAuth(ClientAuth.REQUIRE);
        } else {
            sslContextBuilder = sslContextBuilder.clientAuth(ClientAuth.NONE);
        }
        sslContextBuilder = GrpcSslContexts.configure(sslContextBuilder);
        serverBuilder = serverBuilder.sslContext(sslContextBuilder.build());
    }
    logger.info("Starting gRPC server on port: {}", new Object[]{port.toString()});
    this.server = serverBuilder.build().start();
}
 

private boolean reloadDictionary(final ProcessContext context, final boolean force, final ComponentLog logger) throws IOException {
    boolean obtainedLock;
    if (force) {
        dictionaryUpdateLock.lock();
        obtainedLock = true;
    } else {
        obtainedLock = dictionaryUpdateLock.tryLock();
    }

    if (obtainedLock) {
        try {
            final Search<byte[]> search = new AhoCorasick<>();
            final Set<SearchTerm<byte[]>> terms = new HashSet<>();

            final InputStream inStream = Files.newInputStream(Paths.get(context.getProperty(DICTIONARY).getValue()), StandardOpenOption.READ);

            final TermLoader termLoader;
            if (context.getProperty(DICTIONARY_ENCODING).getValue().equalsIgnoreCase(TEXT_ENCODING)) {
                termLoader = new TextualTermLoader(inStream);
            } else {
                termLoader = new BinaryTermLoader(inStream);
            }

            try {
                SearchTerm<byte[]> term;
                while ((term = termLoader.nextTerm()) != null) {
                    terms.add(term);
                }

                search.initializeDictionary(terms);
                searchRef.set(search);
                logger.info("Loaded search dictionary from {}", new Object[]{context.getProperty(DICTIONARY).getValue()});
                return true;
            } finally {
                termLoader.close();
            }
        } finally {
            dictionaryUpdateLock.unlock();
        }
    } else {
        return false;
    }
}
 

void connectToCassandra(ProcessContext context) {
    if (cluster.get() == null) {
        ComponentLog log = getLogger();
        final String contactPointList = context.getProperty(CONTACT_POINTS).evaluateAttributeExpressions().getValue();
        final String consistencyLevel = context.getProperty(CONSISTENCY_LEVEL).getValue();
        final String compressionType = context.getProperty(COMPRESSION_TYPE).getValue();
        List<InetSocketAddress> contactPoints = getContactPoints(contactPointList);

        // Set up the client for secure (SSL/TLS communications) if configured to do so
        final SSLContextService sslService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
        final String rawClientAuth = context.getProperty(CLIENT_AUTH).getValue();
        final SSLContext sslContext;

        if (sslService != null) {
            final SslContextFactory.ClientAuth clientAuth;

            if (StringUtils.isBlank(rawClientAuth)) {
                clientAuth = SslContextFactory.ClientAuth.REQUIRED;
            } else {
                try {
                    clientAuth = SslContextFactory.ClientAuth.valueOf(rawClientAuth);
                } catch (final IllegalArgumentException iae) {
                    throw new IllegalStateException(String.format("Unrecognized client auth '%s'. Possible values are [%s]",
                            rawClientAuth, StringUtils.join(SslContextFactory.ClientAuth.values(), ", ")));
                }
            }

            sslContext = sslService.createSSLContext(clientAuth);
        } else {
            sslContext = null;
        }

        final String username, password;
        PropertyValue usernameProperty = context.getProperty(USERNAME).evaluateAttributeExpressions();
        PropertyValue passwordProperty = context.getProperty(PASSWORD).evaluateAttributeExpressions();

        if (usernameProperty != null && passwordProperty != null) {
            username = usernameProperty.getValue();
            password = passwordProperty.getValue();
        } else {
            username = null;
            password = null;
        }

        // Create the cluster and connect to it
        Cluster newCluster = createCluster(contactPoints, sslContext, username, password, compressionType);
        PropertyValue keyspaceProperty = context.getProperty(KEYSPACE).evaluateAttributeExpressions();

        final Session newSession;
        // For Java 11, the getValue() call was added so the test could pass
        if (keyspaceProperty != null && keyspaceProperty.getValue() != null) {
            newSession = newCluster.connect(keyspaceProperty.getValue());
        } else {
            newSession = newCluster.connect();
        }

        newCluster.getConfiguration().getQueryOptions().setConsistencyLevel(ConsistencyLevel.valueOf(consistencyLevel));
        Metadata metadata = newCluster.getMetadata();

        log.info("Connected to Cassandra cluster: {}", new Object[]{metadata.getClusterName()});

        cluster.set(newCluster);
        cassandraSession.set(newSession);
    }
}
 

protected void connectToCassandra(ProcessContext context) {
    if (cluster.get() == null) {
        ComponentLog log = getLogger();
        final String contactPointList = context.getProperty(CONTACT_POINTS).getValue();
        final String consistencyLevel = context.getProperty(CONSISTENCY_LEVEL).getValue();
        List<InetSocketAddress> contactPoints = getContactPoints(contactPointList);

        // Set up the client for secure (SSL/TLS communications) if configured to do so
        final SSLContextService sslService =
                context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
        final String rawClientAuth = context.getProperty(CLIENT_AUTH).getValue();
        final SSLContext sslContext;

        if (sslService != null) {
            final SSLContextService.ClientAuth clientAuth;
            if (StringUtils.isBlank(rawClientAuth)) {
                clientAuth = SSLContextService.ClientAuth.REQUIRED;
            } else {
                try {
                    clientAuth = SSLContextService.ClientAuth.valueOf(rawClientAuth);
                } catch (final IllegalArgumentException iae) {
                    throw new ProviderCreationException(String.format("Unrecognized client auth '%s'. Possible values are [%s]",
                            rawClientAuth, StringUtils.join(SslContextFactory.ClientAuth.values(), ", ")));
                }
            }
            sslContext = sslService.createSSLContext(clientAuth);
        } else {
            sslContext = null;
        }

        final String username, password;
        PropertyValue usernameProperty = context.getProperty(USERNAME);
        PropertyValue passwordProperty = context.getProperty(PASSWORD);

        if (usernameProperty != null && passwordProperty != null) {
            username = usernameProperty.getValue();
            password = passwordProperty.getValue();
        } else {
            username = null;
            password = null;
        }

        // Create the cluster and connect to it
        Cluster newCluster = createCluster(contactPoints, sslContext, username, password);
        PropertyValue keyspaceProperty = context.getProperty(KEYSPACE);
        final Session newSession;
        if (keyspaceProperty != null) {
            newSession = newCluster.connect(keyspaceProperty.getValue());
        } else {
            newSession = newCluster.connect();
        }
        newCluster.getConfiguration().getQueryOptions().setConsistencyLevel(ConsistencyLevel.valueOf(consistencyLevel));
        Metadata metadata = newCluster.getMetadata();
        log.info("Connected to Cassandra cluster: {}", new Object[]{metadata.getClusterName()});
        cluster.set(newCluster);
        cassandraSession.set(newSession);
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final Map<String, Pattern> patterns = regexMapRef.get();
    final ComponentLog logger = getLogger();

    final SortedMap<String, String> attributes = getRelevantAttributes(flowFile, patterns);
    if (attributes.size() != patterns.size()) {
        final Set<String> wantedKeys = patterns.keySet();
        final Set<String> foundKeys = attributes.keySet();
        final StringBuilder missingKeys = new StringBuilder();
        for (final String wantedKey : wantedKeys) {
            if (!foundKeys.contains(wantedKey)) {
                missingKeys.append(wantedKey).append(" ");
            }
        }

        logger.error("routing {} to 'failure' because of missing attributes: {}", new Object[]{flowFile, missingKeys.toString()});
        session.transfer(flowFile, REL_FAILURE);
    } else {
        // create single string of attribute key/value pairs to use for group ID hash
        final StringBuilder hashableValue = new StringBuilder();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            hashableValue.append(entry.getKey());
            if (StringUtils.isBlank(entry.getValue())) {
                hashableValue.append("EMPTY");
            } else {
                hashableValue.append(entry.getValue());
            }
        }

        // create group ID
        final String hashValue = DigestUtils.md5Hex(hashableValue.toString());

        logger.info("adding Hash Value {} to attributes for {} and routing to success", new Object[]{hashValue, flowFile});
        flowFile = session.putAttribute(flowFile, context.getProperty(HASH_VALUE_ATTRIBUTE).getValue(), hashValue);
        session.getProvenanceReporter().modifyAttributes(flowFile);
        session.transfer(flowFile, REL_SUCCESS);
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();

    boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
    String encoding = context.getProperty(ENCODING).getValue();
    StreamCallback encoder = null;

    // Select the encoder/decoder to use
    if (encode) {
        if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
            encoder = new EncodeBase64();
        } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
            encoder = new EncodeBase32();
        } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
            encoder = new EncodeHex();
        }
    } else {
        if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
            encoder = new DecodeBase64();
        } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
            encoder = new DecodeBase32();
        } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
            encoder = new DecodeHex();
        }
    }

    if (encoder == null) {
        logger.warn("Unknown operation: {} {}", new Object[]{encode ? "encode" : "decode", encoding});
        return;
    }

    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, encoder);

        logger.info("Successfully {} {}", new Object[]{encode ? "encoded" : "decoded", flowFile});
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (Exception e) {
        logger.error("Failed to {} {} due to {}", new Object[]{encode ? "encode" : "decode", flowFile, e});
        session.transfer(flowFile, REL_FAILURE);
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    List<FlowFile> flowFiles = session.get(new ThrottleFilter(MAX_FLOW_FILES_PER_BATCH));
    if (flowFiles.isEmpty()) {
        context.yield();
        return;
    }

    // Periodically clear any Throttle that has not been used in more than 2 throttling periods
    final long lastClearTime = lastThrottleClearTime.get();
    final long throttleExpirationMillis = System.currentTimeMillis() - 2 * context.getProperty(TIME_PERIOD).asTimePeriod(TimeUnit.MILLISECONDS);
    if (lastClearTime < throttleExpirationMillis) {
        if (lastThrottleClearTime.compareAndSet(lastClearTime, System.currentTimeMillis())) {
            final Iterator<Map.Entry<String, Throttle>> itr = throttleMap.entrySet().iterator();
            while (itr.hasNext()) {
                final Map.Entry<String, Throttle> entry = itr.next();
                final Throttle throttle = entry.getValue();
                if (throttle.tryLock()) {
                    try {
                        if (throttle.lastUpdateTime() < lastClearTime) {
                            itr.remove();
                        }
                    } finally {
                        throttle.unlock();
                    }
                }
            }
        }
    }

    final ComponentLog logger = getLogger();
    for (FlowFile flowFile : flowFiles) {
        // call this to capture potential error
        final long accrualAmount = getFlowFileAccrual(flowFile);
        if (accrualAmount < 0) {
            logger.error("Routing {} to 'failure' due to missing or invalid attribute", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
        } else {
            logger.info("transferring {} to 'success'", new Object[]{flowFile});
            session.transfer(flowFile, REL_SUCCESS);
        }
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();

    final Charset inputCharset = Charset.forName(context.getProperty(INPUT_CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    final Charset outputCharset = Charset.forName(context.getProperty(OUTPUT_CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    final CharBuffer charBuffer = CharBuffer.allocate(MAX_BUFFER_SIZE);

    final CharsetDecoder decoder = inputCharset.newDecoder();
    decoder.onMalformedInput(CodingErrorAction.REPLACE);
    decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
    decoder.replaceWith("?");

    final CharsetEncoder encoder = outputCharset.newEncoder();
    encoder.onMalformedInput(CodingErrorAction.REPLACE);
    encoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
    encoder.replaceWith("?".getBytes(outputCharset));

    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, new StreamCallback() {
            @Override
            public void process(final InputStream rawIn, final OutputStream rawOut) throws IOException {
                try (final BufferedReader reader = new BufferedReader(new InputStreamReader(rawIn, decoder), MAX_BUFFER_SIZE);
                        final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(rawOut, encoder), MAX_BUFFER_SIZE)) {
                    int charsRead;
                    while ((charsRead = reader.read(charBuffer)) != -1) {
                        charBuffer.flip();
                        writer.write(charBuffer.array(), 0, charsRead);
                    }

                    writer.flush();
                }
            }
        });

        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        logger.info("successfully converted characters from {} to {} for {}",
                new Object[]{inputCharset, outputCharset, flowFile});
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final Exception e) {
        throw new ProcessException(e);
    }
}
 

/**
 * Configures connection pool by creating an instance of the
 * {@link BasicDataSource} based on configuration provided with
 * {@link ConfigurationContext}.
 * <p>
 * This operation makes no guarantees that the actual connection could be
 * made since the underlying system may still go off-line during normal
 * operation of the connection pool.
 *
 * @param context the configuration context
 * @throws InitializationException if unable to create a database connection
 */
@OnEnabled
public void onConfigured(final ConfigurationContext context) throws InitializationException {

    connectionUrl = context.getProperty(DATABASE_URL).getValue();

    ComponentLog log = getLogger();

    final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).getValue();
    final Configuration hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles);
    final String validationQuery = context.getProperty(VALIDATION_QUERY).evaluateAttributeExpressions().getValue();

    // add any dynamic properties to the Hive configuration
    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        final PropertyDescriptor descriptor = entry.getKey();
        if (descriptor.isDynamic()) {
            hiveConfig.set(descriptor.getName(), entry.getValue());
        }
    }

    final String drv = HiveDriver.class.getName();
    if (SecurityUtil.isSecurityEnabled(hiveConfig)) {
        final String principal = context.getProperty(kerberosProperties.getKerberosPrincipal()).getValue();
        final String keyTab = context.getProperty(kerberosProperties.getKerberosKeytab()).getValue();

        log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[]{principal, keyTab});
        try {
            ugi = hiveConfigurator.authenticate(hiveConfig, principal, keyTab, TICKET_RENEWAL_PERIOD, log);
        } catch (AuthenticationFailedException ae) {
            log.error(ae.getMessage(), ae);
        }
        getLogger().info("Successfully logged in as principal {} with keytab {}", new Object[]{principal, keyTab});

    }
    final String user = context.getProperty(DB_USER).getValue();
    final String passw = context.getProperty(DB_PASSWORD).getValue();
    final Long maxWaitMillis = context.getProperty(MAX_WAIT_TIME).asTimePeriod(TimeUnit.MILLISECONDS);
    final Integer maxTotal = context.getProperty(MAX_TOTAL_CONNECTIONS).asInteger();

    dataSource = new BasicDataSource();
    dataSource.setDriverClassName(drv);

    final String dburl = context.getProperty(DATABASE_URL).getValue();

    dataSource.setMaxWait(maxWaitMillis);
    dataSource.setMaxActive(maxTotal);

    if (validationQuery != null && !validationQuery.isEmpty()) {
        dataSource.setValidationQuery(validationQuery);
        dataSource.setTestOnBorrow(true);
    }

    dataSource.setUrl(dburl);
    dataSource.setUsername(user);
    dataSource.setPassword(passw);
}
 

@Override
protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) {
    final Configuration conf = getConfiguration();
    final FileSystem hdfs = getFileSystem();
    final String flowFileContentValue = context.getProperty(FLOWFILE_CONTENT).getValue();
    final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean();
    final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B);
    if (bufferSizeProp != null) {
        int bufferSize = bufferSizeProp.intValue();
        conf.setInt(BUFFER_SIZE_KEY, bufferSize);
    }
    ComponentLog logger = getLogger();
    final SequenceFileReader<Set<FlowFile>> reader;
    if (flowFileContentValue.equalsIgnoreCase(VALUE_ONLY)) {
        reader = new ValueReader(session);
    } else {
        reader = new KeyValueReader(session);
    }
    Set<FlowFile> flowFiles = Collections.emptySet();
    for (final Path file : files) {
        if (!this.isScheduled()) {
            break; // This processor should stop running immediately.
        }

        final StopWatch stopWatch = new StopWatch(false);
        try {
            stopWatch.start();
            if (!hdfs.exists(file)) {
                continue; // If file is no longer here move on.
            }
            logger.debug("Reading file");
            flowFiles = getFlowFiles(conf, hdfs, reader, file);
            if (!keepSourceFiles && !hdfs.delete(file, false)) {
                logger.warn("Unable to delete path " + file.toString() + " from HDFS.  Will likely be picked up over and over...");
            }
        } catch (Throwable t) {
            logger.error("Error retrieving file {} from HDFS due to {}", new Object[]{file, t});
            session.rollback();
            context.yield();
        } finally {
            stopWatch.stop();
            long totalSize = 0;
            for (FlowFile flowFile : flowFiles) {
                totalSize += flowFile.getSize();
                session.getProvenanceReporter().receive(flowFile, file.toString());
            }
            if (totalSize > 0) {
                final String dataRate = stopWatch.calculateDataRate(totalSize);
                final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
                logger.info("Created {} flowFiles from SequenceFile {}. Ingested in {} milliseconds at a rate of {}", new Object[]{
                    flowFiles.size(), file.toUri().toASCIIString(), millis, dataRate});
                logger.info("Transferred flowFiles {}  to success", new Object[]{flowFiles});
                session.transfer(flowFiles, REL_SUCCESS);
            }
        }
    }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();
    PackageFormat packagingFormat = PackageFormat.getFormat(context.getProperty(PACKAGING_FORMAT).getValue().toLowerCase());
    if (packagingFormat == PackageFormat.AUTO_DETECT_FORMAT) {
        packagingFormat = null;
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No mime.type attribute set for {}; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        for (PackageFormat format: PackageFormat.values()) {
            if (mimeType.toLowerCase().equals(format.getMimeType())) {
                packagingFormat = format;
            }
        }
        if (packagingFormat == null) {
            logger.info("Cannot unpack {} because its mime.type attribute is set to '{}', which is not a format that can be unpacked; routing to 'success'", new Object[]{flowFile, mimeType});
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }

    // set the Unpacker to use for this FlowFile.  FlowFileUnpackager objects maintain state and are not reusable.
    final Unpacker unpacker;
    final boolean addFragmentAttrs;
    switch (packagingFormat) {
    case TAR_FORMAT:
    case X_TAR_FORMAT:
        unpacker = tarUnpacker;
        addFragmentAttrs = true;
        break;
    case ZIP_FORMAT:
        unpacker = zipUnpacker;
        addFragmentAttrs = true;
        break;
    case FLOWFILE_STREAM_FORMAT_V2:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV2());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_STREAM_FORMAT_V3:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV3());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_TAR_FORMAT:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV1());
        addFragmentAttrs = false;
        break;
    case AUTO_DETECT_FORMAT:
    default:
        // The format of the unpacker should be known before initialization
        throw new ProcessException(packagingFormat + " is not a valid packaging format");
    }

    final List<FlowFile> unpacked = new ArrayList<>();
    try {
        unpacker.unpack(session, flowFile, unpacked);
        if (unpacked.isEmpty()) {
            logger.error("Unable to unpack {} because it does not appear to have any entries; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        if (addFragmentAttrs) {
            finishFragmentAttributes(session, flowFile, unpacked);
        }
        session.transfer(unpacked, REL_SUCCESS);
        final String fragmentId = unpacked.size() > 0 ? unpacked.get(0).getAttribute(FRAGMENT_ID) : null;
        flowFile = FragmentAttributes.copyAttributesToOriginal(session, flowFile, fragmentId, unpacked.size());
        session.transfer(flowFile, REL_ORIGINAL);
        session.getProvenanceReporter().fork(flowFile, unpacked);
        logger.info("Unpacked {} into {} and transferred to success", new Object[]{flowFile, unpacked});
    } catch (final Exception e) {
        logger.error("Unable to unpack {} due to {}; routing to failure", new Object[]{flowFile, e});
        session.transfer(flowFile, REL_FAILURE);
        session.remove(unpacked);
    }
}