Java Code Examples for org.apache.nifi.remote.client.SiteToSiteClientConfig#getPreferredBatchDuration()

The following examples show how to use org.apache.nifi.remote.client.SiteToSiteClientConfig#getPreferredBatchDuration() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SiteToSiteCliMainTest.java From localization_nifi with Apache License 2.0

6 votes

@Before
public void setup() {
    SiteToSiteClient.Builder builder = new SiteToSiteClient.Builder();
    expectedUrl = SiteToSiteCliMain.URL_OPTION_DEFAULT;
    expectedTransferDirection = TransferDirection.valueOf(SiteToSiteCliMain.DIRECTION_OPTION_DEFAULT);
    expectedSiteToSiteTransportProtocol = SiteToSiteTransportProtocol.valueOf(SiteToSiteCliMain.TRANSPORT_PROTOCOL_OPTION_DEFAULT);
    expectedPortName = builder.getPortName();
    expectedPortIdentifier = builder.getPortIdentifier();
    expectedTimeoutNs = builder.getTimeout(TimeUnit.NANOSECONDS);
    expectedPenalizationNs = builder.getPenalizationPeriod(TimeUnit.NANOSECONDS);
    expectedKeystoreFilename = builder.getKeystoreFilename();
    expectedKeystorePass = builder.getKeystorePass();
    expectedKeystoreType = builder.getKeystoreType();
    expectedTruststoreFilename = builder.getTruststoreFilename();
    expectedTruststorePass = builder.getTruststorePass();
    expectedTruststoreType = builder.getTruststoreType();
    expectedCompression = false;
    expectedPeerPersistenceFile = builder.getPeerPersistenceFile();
    SiteToSiteClientConfig siteToSiteClientConfig = builder.buildConfig();
    expectedBatchCount = siteToSiteClientConfig.getPreferredBatchCount();
    expectedBatchDuration = siteToSiteClientConfig.getPreferredBatchDuration(TimeUnit.NANOSECONDS);
    expectedBatchSize = siteToSiteClientConfig.getPreferredBatchSize();
    expectedHttpProxy = siteToSiteClientConfig.getHttpProxy();
}

Example 2

Source File: SiteToSiteCliMainTest.java From nifi with Apache License 2.0

6 votes

@Before
public void setup() {
    SiteToSiteClient.Builder builder = new SiteToSiteClient.Builder();
    expectedUrl = SiteToSiteCliMain.URL_OPTION_DEFAULT;
    expectedTransferDirection = TransferDirection.valueOf(SiteToSiteCliMain.DIRECTION_OPTION_DEFAULT);
    expectedSiteToSiteTransportProtocol = SiteToSiteTransportProtocol.valueOf(SiteToSiteCliMain.TRANSPORT_PROTOCOL_OPTION_DEFAULT);
    expectedPortName = builder.getPortName();
    expectedPortIdentifier = builder.getPortIdentifier();
    expectedTimeoutNs = builder.getTimeout(TimeUnit.NANOSECONDS);
    expectedPenalizationNs = builder.getPenalizationPeriod(TimeUnit.NANOSECONDS);
    expectedKeystoreFilename = builder.getKeystoreFilename();
    expectedKeystorePass = builder.getKeystorePass();
    expectedKeystoreType = builder.getKeystoreType();
    expectedTruststoreFilename = builder.getTruststoreFilename();
    expectedTruststorePass = builder.getTruststorePass();
    expectedTruststoreType = builder.getTruststoreType();
    expectedCompression = false;
    expectedPeerPersistenceFile = builder.getPeerPersistenceFile();
    SiteToSiteClientConfig siteToSiteClientConfig = builder.buildConfig();
    expectedBatchCount = siteToSiteClientConfig.getPreferredBatchCount();
    expectedBatchDuration = siteToSiteClientConfig.getPreferredBatchDuration(TimeUnit.NANOSECONDS);
    expectedBatchSize = siteToSiteClientConfig.getPreferredBatchSize();
    expectedHttpProxy = siteToSiteClientConfig.getHttpProxy();
}

Example 3

Source File: StandardRemoteGroupPort.java From nifi with Apache License 2.0

4 votes

private int transferFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session, final FlowFile firstFlowFile) throws IOException, ProtocolException {
    FlowFile flowFile = firstFlowFile;

    try {
        final String userDn = transaction.getCommunicant().getDistinguishedName();
        final long startSendingNanos = System.nanoTime();
        final StopWatch stopWatch = new StopWatch(true);
        long bytesSent = 0L;

        final SiteToSiteClientConfig siteToSiteClientConfig = getSiteToSiteClient().getConfig();
        final long maxBatchBytes = siteToSiteClientConfig.getPreferredBatchSize();
        final int maxBatchCount = siteToSiteClientConfig.getPreferredBatchCount();
        final long preferredBatchDuration = siteToSiteClientConfig.getPreferredBatchDuration(TimeUnit.NANOSECONDS);
        final long maxBatchDuration = preferredBatchDuration > 0 ? preferredBatchDuration : BATCH_SEND_NANOS;


        final Set<FlowFile> flowFilesSent = new HashSet<>();
        boolean continueTransaction = true;
        while (continueTransaction) {
            final long startNanos = System.nanoTime();
            // call codec.encode within a session callback so that we have the InputStream to read the FlowFile
            final FlowFile toWrap = flowFile;
            session.read(flowFile, new InputStreamCallback() {
                @Override
                public void process(final InputStream in) throws IOException {
                    final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize());
                    transaction.send(dataPacket);
                }
            });

            final long transferNanos = System.nanoTime() - startNanos;
            final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);

            flowFilesSent.add(flowFile);
            bytesSent += flowFile.getSize();
            logger.debug("{} Sent {} to {}", this, flowFile, transaction.getCommunicant().getUrl());

            final String transitUri = transaction.getCommunicant().createTransitUri(flowFile.getAttribute(CoreAttributes.UUID.key()));
            flowFile = session.putAttribute(flowFile, SiteToSiteAttributes.S2S_PORT_ID.key(), getTargetIdentifier());
            session.getProvenanceReporter().send(flowFile, transitUri, "Remote DN=" + userDn, transferMillis, false);
            session.remove(flowFile);

            final long sendingNanos = System.nanoTime() - startSendingNanos;

            if (maxBatchCount > 0 && flowFilesSent.size() >= maxBatchCount) {
                flowFile = null;
            } else if (maxBatchBytes > 0 && bytesSent >= maxBatchBytes) {
                flowFile = null;
            } else if (sendingNanos >= maxBatchDuration) {
                flowFile = null;
            } else {
                flowFile = session.get();
            }

            continueTransaction = (flowFile != null);
        }

        transaction.confirm();

        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        stopWatch.stop();
        final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
        final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
        final String dataSize = FormatUtils.formatDataSize(bytesSent);

        transaction.complete();
        session.commit();

        final String flowFileDescription = (flowFilesSent.size() < 20) ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
        logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
            this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});

        return flowFilesSent.size();
    } catch (final Exception e) {
        session.rollback();
        throw e;
    }

}