org.apache.nifi.remote.client.SiteToSiteClient Java Examples

The following examples show how to use org.apache.nifi.remote.client.SiteToSiteClient. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestHttpClient.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReceiveTimeout() throws Exception {

    try (
            SiteToSiteClient client = getDefaultBuilder()
                    .timeout(1, TimeUnit.SECONDS)
                    .portName("output-timeout")
                    .build()
    ) {
        try {
            client.createTransaction(TransferDirection.RECEIVE);
            fail();
        } catch (IOException e) {
            logger.info("An exception was thrown as expected.", e);
            assertTrue(e instanceof SocketTimeoutException);
        }
    }
}
Example #2
Source File: TestSiteToSiteBulletinReportingTask.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Override
public void setup(ReportingContext reportContext) throws IOException {
    if(siteToSiteClient == null) {
        final SiteToSiteClient client = Mockito.mock(SiteToSiteClient.class);
        final Transaction transaction = Mockito.mock(Transaction.class);

        try {
            Mockito.doAnswer((Answer<Object>) invocation -> {
                final byte[] data = invocation.getArgument(0, byte[].class);
                dataSent.add(data);
                return null;
            }).when(transaction).send(Mockito.any(byte[].class), Mockito.any(Map.class));

            when(client.createTransaction(Mockito.any(TransferDirection.class))).thenReturn(transaction);
        } catch (final Exception e) {
            e.printStackTrace();
            Assert.fail(e.toString());
        }
        siteToSiteClient = client;
    }
}
Example #3
Source File: NiFiSinkMain.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
            .url("http://localhost:8080/nifi")
            .portName("Data from Flink")
            .buildConfig();

    DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five", "q")
            .addSink(new NiFiSink<>(clientConfig, new NiFiDataPacketBuilder<String>() {
                @Override
                public NiFiDataPacket createNiFiDataPacket(String s, RuntimeContext ctx) {
                    return new StandardNiFiDataPacket(s.getBytes(ConfigConstants.DEFAULT_CHARSET),
                            new HashMap<String, String>());
                }
            }));

    env.execute();
}
Example #4
Source File: NiFiSourceMain.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
            .url("http://localhost:8080/nifi")
            .portName("Data for Flink")
            .requestBatchCount(5)
            .buildConfig();

    SourceFunction<NiFiDataPacket> nifiSource = new NiFiSource(clientConfig);
    DataStream<NiFiDataPacket> streamSource = env.addSource(nifiSource).setParallelism(2);

    DataStream<String> dataStream = streamSource.map(new MapFunction<NiFiDataPacket, String>() {
        @Override
        public String map(NiFiDataPacket value) throws Exception {
            return new String(value.getContent(), Charset.defaultCharset());
        }
    });

    dataStream.print();
    env.execute();
}
Example #5
Source File: TestHttpClient.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSendSuccessMultipleUrls() throws Exception {

    final Set<String> urls = new LinkedHashSet<>();
    urls.add("http://localhost:9999");
    urls.add("http://localhost:" + httpConnector.getLocalPort() + "/nifi");

    try (
            final SiteToSiteClient client = getDefaultBuilder()
                    .urls(urls)
                    .portName("input-running")
                    .build()
    ) {
        testSend(client);
    }

}
Example #6
Source File: NiFiSourceTopologyExample.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
			.url("http://localhost:8080/nifi")
			.portName("Data for Flink")
			.requestBatchCount(5)
			.buildConfig();

	SourceFunction<NiFiDataPacket> nifiSource = new NiFiSource(clientConfig);
	DataStream<NiFiDataPacket> streamSource = env.addSource(nifiSource).setParallelism(2);

	DataStream<String> dataStream = streamSource.map(new MapFunction<NiFiDataPacket, String>() {
		@Override
		public String map(NiFiDataPacket value) throws Exception {
			return new String(value.getContent(), Charset.defaultCharset());
		}
	});

	dataStream.print();
	env.execute();
}
Example #7
Source File: StatelessRemoteInputPort.java    From nifi with Apache License 2.0 6 votes vote down vote up 
public StatelessRemoteInputPort(final VersionedRemoteProcessGroup rpg, final VersionedRemoteGroupPort remotePort, final SSLContext sslContext) {
    final String timeout = rpg.getCommunicationsTimeout();
    final long timeoutMillis = FormatUtils.getTimeDuration(timeout, TimeUnit.MILLISECONDS);

    url = rpg.getTargetUris();
    name = remotePort.getName();

    client = new SiteToSiteClient.Builder()
        .portName(remotePort.getName())
        .timeout(timeoutMillis, TimeUnit.MILLISECONDS)
        .transportProtocol(SiteToSiteTransportProtocol.valueOf(rpg.getTransportProtocol()))
        .url(rpg.getTargetUris())
        .useCompression(remotePort.isUseCompression())
        .sslContext(sslContext)
        .eventReporter(EventReporter.NO_OP)
        .build();
}
Example #8
Source File: NiFiSinkMain.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
            .url("http://localhost:8080/nifi")
            .portName("Data from Flink")
            .buildConfig();

    DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five", "q")
            .addSink(new NiFiSink<>(clientConfig, new NiFiDataPacketBuilder<String>() {
                @Override
                public NiFiDataPacket createNiFiDataPacket(String s, RuntimeContext ctx) {
                    return new StandardNiFiDataPacket(s.getBytes(ConfigConstants.DEFAULT_CHARSET),
                            new HashMap<String, String>());
                }
            }));

    env.execute();
}
Example #9
Source File: NiFiSinkTopologyExample.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
			.url("http://localhost:8080/nifi")
			.portName("Data from Flink")
			.buildConfig();

	DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five", "q")
			.addSink(new NiFiSink<>(clientConfig, new NiFiDataPacketBuilder<String>() {
				@Override
				public NiFiDataPacket createNiFiDataPacket(String s, RuntimeContext ctx) {
					return new StandardNiFiDataPacket(s.getBytes(ConfigConstants.DEFAULT_CHARSET),
						new HashMap<String, String>());
				}
			}));

	env.execute();
}
Example #10
Source File: StandardRemoteGroupPort.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Override
public void onSchedulingStart() {
    super.onSchedulingStart();

    final long penalizationMillis = FormatUtils.getTimeDuration(remoteGroup.getYieldDuration(), TimeUnit.MILLISECONDS);

    final SiteToSiteClient client = new SiteToSiteClient.Builder()
            .urls(SiteToSiteRestApiClient.parseClusterUrls(remoteGroup.getTargetUris()))
            .portIdentifier(getIdentifier())
            .sslContext(sslContext)
            .useCompression(isUseCompression())
            .eventReporter(remoteGroup.getEventReporter())
            .peerPersistenceFile(getPeerPersistenceFile(getIdentifier(), nifiProperties))
            .nodePenalizationPeriod(penalizationMillis, TimeUnit.MILLISECONDS)
            .timeout(remoteGroup.getCommunicationsTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS)
            .transportProtocol(remoteGroup.getTransportProtocol())
            .httpProxy(new HttpProxy(remoteGroup.getProxyHost(), remoteGroup.getProxyPort(), remoteGroup.getProxyUser(), remoteGroup.getProxyPassword()))
            .localAddress(remoteGroup.getLocalAddress())
            .build();
    clientRef.set(client);
}
Example #11
Source File: TestHttpClient.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNoAvailablePeer() throws Exception {

    peers = new HashSet<>();

    try (
        SiteToSiteClient client = getDefaultBuilder()
            .portName("input-running")
            .build()
    ) {
        final Transaction transaction = client.createTransaction(TransferDirection.SEND);

        assertNull(transaction);

    }

}
Example #12
Source File: TestSiteToSiteProvenanceReportingTask.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Override
protected SiteToSiteClient getClient() {
    final SiteToSiteClient client = Mockito.mock(SiteToSiteClient.class);
    final Transaction transaction = Mockito.mock(Transaction.class);

    try {
        Mockito.doAnswer(new Answer<Object>() {
            @Override
            public Object answer(final InvocationOnMock invocation) throws Throwable {
                final byte[] data = invocation.getArgumentAt(0, byte[].class);
                dataSent.add(data);
                return null;
            }
        }).when(transaction).send(Mockito.any(byte[].class), Mockito.any(Map.class));

        Mockito.when(client.createTransaction(Mockito.any(TransferDirection.class))).thenReturn(transaction);
    } catch (final Exception e) {
        e.printStackTrace();
        Assert.fail(e.toString());
    }

    return client;
}
Example #13
Source File: TestHttpClient.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSendAccessDeniedHTTPS() throws Exception {

    try (
            final SiteToSiteClient client = getDefaultBuilderHTTPS()
                    .portName("input-access-denied")
                    .build()
    ) {
        try {
            client.createTransaction(TransferDirection.SEND);
            fail("Handshake exception should be thrown.");
        } catch (HandshakeException e) {
        }
    }

}
Example #14
Source File: TestSiteToSiteReportingRecordSink.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Override
protected SiteToSiteClient getClient() {
    final SiteToSiteClient client = Mockito.mock(SiteToSiteClient.class);
    final Transaction transaction = Mockito.mock(Transaction.class);

    try {
        Mockito.doAnswer((Answer<Object>) invocation -> {
            final byte[] data = invocation.getArgument(0, byte[].class);
            dataSent.add(data);
            return null;
        }).when(transaction).send(Mockito.any(byte[].class), Mockito.any(Map.class));

        Mockito.when(client.createTransaction(Mockito.any(TransferDirection.class))).thenReturn(transaction);
    } catch (final Exception e) {
        e.printStackTrace();
        Assert.fail(e.toString());
    }

    return client;
}
Example #15
Source File: TestHttpClient.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSendUnknownPort() throws Exception {

    try (
        SiteToSiteClient client = getDefaultBuilder()
            .portName("input-unknown")
            .build()
    ) {
        try {
            client.createTransaction(TransferDirection.SEND);
            fail();
        } catch (IOException e) {
            logger.info("Exception message: {}", e.getMessage());
            assertTrue(e.getMessage().contains("Failed to determine the identifier of port"));
        }
    }
}
Example #16
Source File: TestHttpClient.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReceiveUnknownPort() throws Exception {

    try (
        SiteToSiteClient client = getDefaultBuilder()
            .portName("output-unknown")
            .build()
    ) {
        try {
            client.createTransaction(TransferDirection.RECEIVE);
            fail();
        } catch (IOException e) {
            logger.info("Exception message: {}", e.getMessage());
            assertTrue(e.getMessage().contains("Failed to determine the identifier of port"));
        }
    }
}
Example #17
Source File: TestSiteToSiteMetricsReportingTask.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Override
public void setup(ReportingContext reportContext) throws IOException {
    if(siteToSiteClient == null) {
        final SiteToSiteClient client = Mockito.mock(SiteToSiteClient.class);
        final Transaction transaction = Mockito.mock(Transaction.class);

        try {
            Mockito.doAnswer((Answer<Object>) invocation -> {
                final byte[] data = invocation.getArgument(0, byte[].class);
                dataSent.add(data);
                return null;
            }).when(transaction).send(Mockito.any(byte[].class), Mockito.any(Map.class));

            when(client.createTransaction(Mockito.any(TransferDirection.class))).thenReturn(transaction);
        } catch (final Exception e) {
            e.printStackTrace();
            Assert.fail(e.toString());
        }
        siteToSiteClient = client;
    }
}
Example #18
Source File: TestHttpClient.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReceiveUnknownPort() throws Exception {

    try (
        SiteToSiteClient client = getDefaultBuilder()
            .portName("output-unknown")
            .build()
    ) {
        try {
            client.createTransaction(TransferDirection.RECEIVE);
            fail();
        } catch (IOException e) {
            logger.info("Exception message: {}", e.getMessage());
            assertTrue(e.getMessage().contains("Failed to determine the identifier of port"));
        }
    }
}
Example #19
Source File: TestHttpClient.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReceiveSlowClientSuccess() throws Exception {

    try (
            SiteToSiteClient client = getDefaultBuilder()
                    .portName("output-running")
                    .build()
    ) {
        final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);

        assertNotNull(transaction);

        DataPacket packet;
        while ((packet = transaction.receive()) != null) {
            consumeDataPacket(packet);
            Thread.sleep(500);
        }
        transaction.confirm();
        transaction.complete();
    }
}
Example #20
Source File: TestHttpClient.java    From nifi with Apache License 2.0 6 votes vote down vote up 
private void testSend(SiteToSiteClient client) throws Exception {

        testSendIgnoreProxyError(client, transaction -> {
            serverChecksum = "1071206772";

            for (int i = 0; i < 20; i++) {
                DataPacket packet = new DataPacketBuilder()
                        .contents("Example contents from client.")
                        .attr("Client attr 1", "Client attr 1 value")
                        .attr("Client attr 2", "Client attr 2 value")
                        .build();
                transaction.send(packet);
                long written = ((Peer)transaction.getCommunicant()).getCommunicationsSession().getBytesWritten();
                logger.info("{}: {} bytes have been written.", i, written);
            }
        });

    }
Example #21
Source File: TestHttpClient.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReceiveTimeout() throws Exception {

    try (
            SiteToSiteClient client = getDefaultBuilder()
                    .timeout(1, TimeUnit.SECONDS)
                    .portName("output-timeout")
                    .build()
    ) {
        try {
            client.createTransaction(TransferDirection.RECEIVE);
            fail();
        } catch (IOException e) {
            logger.info("An exception was thrown as expected.", e);
            assertTrue(e instanceof SocketTimeoutException);
        }
    }
}
Example #22
Source File: TestHttpClient.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSendSuccessMultipleUrls() throws Exception {

    final Set<String> urls = new LinkedHashSet<>();
    urls.add("http://localhost:9999");
    urls.add("http://localhost:" + httpConnector.getLocalPort() + "/nifi");

    try (
            final SiteToSiteClient client = getDefaultBuilder()
                    .urls(urls)
                    .portName("input-running")
                    .build()
    ) {
        testSend(client);
    }

}
Example #23
Source File: TestHttpClient.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNoAvailablePeer() throws Exception {

    peers = new HashSet<>();

    try (
        SiteToSiteClient client = getDefaultBuilder()
            .portName("input-running")
            .build()
    ) {
        final Transaction transaction = client.createTransaction(TransferDirection.SEND);

        assertNull(transaction);

    }

}
Example #24
Source File: SiteToSiteCliMain.java    From nifi with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) {
    // Make IO redirection useful
    PrintStream output = System.out;
    System.setOut(System.err);
    Options options = new Options();
    try {
        CliParse cliParse = parseCli(options, args);
        try (SiteToSiteClient siteToSiteClient = cliParse.getBuilder().build()) {
            if (cliParse.getTransferDirection() == TransferDirection.SEND) {
                new SiteToSiteSender(siteToSiteClient, System.in).sendFiles();
            } else {
                new SiteToSiteReceiver(siteToSiteClient, output).receiveFiles();
            }
        }
    } catch (Exception e) {
        printUsage(e.getMessage(), options);
        e.printStackTrace();
    }
}
Example #25
Source File: StandardRemoteGroupPort.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Override
public void shutdown() {
    super.shutdown();

    final SiteToSiteClient client = getSiteToSiteClient();
    if (client != null) {
        try {
            client.close();
        } catch (final IOException ioe) {
            logger.warn("Failed to properly shutdown Site-to-Site Client due to {}", ioe);
        }
    }
}
Example #26
Source File: TestStandardRemoteGroupPort.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
private void setupMock(final SiteToSiteTransportProtocol protocol,
        final TransferDirection direction,
        final Transaction transaction) throws Exception {
    processGroup = null;
    remoteGroup = mock(RemoteProcessGroup.class);
    scheduler = null;
    siteToSiteClient = mock(SiteToSiteClient.class);
    this.transaction = transaction;

    eventReporter = mock(EventReporter.class);

    final ConnectableType connectableType;
    switch (direction) {
        case SEND:
            connectableType = ConnectableType.REMOTE_INPUT_PORT;
            break;
        case RECEIVE:
            connectableType = ConnectableType.OUTPUT_PORT;
            break;
        default:
            connectableType = null;
            break;
    }

    port = spy(new StandardRemoteGroupPort(ID, NAME,
            processGroup, remoteGroup, direction, connectableType, null, scheduler, NiFiProperties.createBasicNiFiProperties(null, null)));

    doReturn(true).when(remoteGroup).isTransmitting();
    doReturn(protocol).when(remoteGroup).getTransportProtocol();
    doReturn(REMOTE_CLUSTER_URL).when(remoteGroup).getTargetUri();
    doReturn(siteToSiteClient).when(port).getSiteToSiteClient();
    doReturn(transaction).when(siteToSiteClient).createTransaction(eq(direction));
    doReturn(eventReporter).when(remoteGroup).getEventReporter();

}
Example #27
Source File: TestHttpClient.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReceiveSuccessHTTPS() throws Exception {

    try (
            SiteToSiteClient client = getDefaultBuilderHTTPS()
                    .portName("output-running")
                    .build()
    ) {
        testReceive(client);
    }
}
Example #28
Source File: TestHttpClient.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReceiveSuccessWithProxyAuth() throws Exception {

    try (
            SiteToSiteClient client = getDefaultBuilder()
                    .portName("output-running")
                    .httpProxy(new HttpProxy("localhost", proxyServerWithAuth.getListenAddress().getPort(), PROXY_USER, PROXY_PASSWORD))
                    .build()
    ) {
        testReceive(client);
    }
}
Example #29
Source File: TestHttpClient.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
private void testReceive(SiteToSiteClient client) throws IOException {
    final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);

    assertNotNull(transaction);

    DataPacket packet;
    while ((packet = transaction.receive()) != null) {
        consumeDataPacket(packet);
    }
    transaction.confirm();
    transaction.complete();
}
Example #30
Source File: NiFiSinglePortOutputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
/**
 * @param siteToSiteBuilder the builder for a NiFi SiteToSiteClient
 * @param dataPacketBuilder a builder to produce NiFiDataPackets from incoming data
 * @param windowDataManager  a WindowDataManager to save and load state for windows of tuples
 * @param batchSize the maximum number of tuples to send to NiFi in a single transaction
 */
public NiFiSinglePortOutputOperator(
    final SiteToSiteClient.Builder siteToSiteBuilder,
    final NiFiDataPacketBuilder<T> dataPacketBuilder,
    final WindowDataManager windowDataManager,
    final int batchSize)
{
  super(siteToSiteBuilder, dataPacketBuilder, windowDataManager);
  this.inputPort = new BufferingInputPort(batchSize);
}