Java Code Examples for org.apache.flume.source.avro.AvroFlumeEvent#setHeaders()

@Test
public void flumeAvroIpc() throws StageException, IOException, ExecutionException, InterruptedException {

  final Charset charset = Charsets.UTF_8;
  final TCPServerSourceConfig configBean = createConfigBean(charset);
  configBean.tcpMode = TCPMode.FLUME_AVRO_IPC;
  configBean.dataFormat = DataFormat.TEXT;
  configBean.bindAddress = "0.0.0.0";

  final int batchSize = 5;
  final String outputLane = "output";

  final TCPServerSource source = new TCPServerSource(configBean);
  final PushSourceRunner runner = new PushSourceRunner.Builder(TCPServerDSource.class, source)
      .addOutputLane(outputLane)
      .setOnRecordError(OnRecordError.TO_ERROR)
      .build();

  runner.runInit();

  runner.runProduce(Collections.emptyMap(), batchSize, out -> {
    final Map<String, List<Record>> outputMap = out.getRecords();
    assertThat(outputMap, hasKey(outputLane));
    final List<Record> records = outputMap.get(outputLane);
    assertThat(records, hasSize(batchSize));
    for (int i = 0; i < batchSize; i++) {
      assertThat(
          records.get(i).get("/" + TextDataParserFactory.TEXT_FIELD_NAME),
          fieldWithValue(getFlumeAvroIpcEventName(i))
      );
    }
    runner.setStop();
  });

  final AvroSourceProtocol client = SpecificRequestor.getClient(AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress("localhost", Integer.parseInt(configBean.ports.get(0)))));

  List<AvroFlumeEvent> events = new LinkedList<>();
  for (int i = 0; i < batchSize; i++) {
    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
    avroEvent.setBody(ByteBuffer.wrap(getFlumeAvroIpcEventName(i).getBytes()));
    events.add(avroEvent);
  }

  Status status = client.appendBatch(events);

  assertThat(status, equalTo(Status.OK));

  runner.waitOnProduce();
}
 

private void doRequest(boolean serverEnableCompression, boolean clientEnableCompression, int compressionLevel) throws InterruptedException, IOException {
  boolean bound = false;

  for (int i = 0; i < 100 && !bound; i++) {
    try {
      Context context = new Context();
      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");
      context.put("threads", "50");
      if (serverEnableCompression) {
        context.put("compression-type", "deflate");
      } else {
        context.put("compression-type", "none");
      }

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  AvroSourceProtocol client;
  if (clientEnableCompression) {
    client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort), new CompressionChannelFactory(6)));
  } else {
    client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort)));
  }

  AvroFlumeEvent avroEvent = new AvroFlumeEvent();

  avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
  avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));

  Status status = client.append(avroEvent);

  Assert.assertEquals(Status.OK, status);

  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "Hello avro",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  logger.debug("Round trip event:{}", event);

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

@Test
public void testSslRequest() throws InterruptedException, IOException {
  boolean bound = false;

  for (int i = 0; i < 10 && !bound; i++) {
    try {
      Context context = new Context();

      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");
      context.put("ssl", "true");
      context.put("keystore", "src/test/resources/server.p12");
      context.put("keystore-password", "password");
      context.put("keystore-type", "PKCS12");

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
      Thread.sleep(100);
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  AvroSourceProtocol client = SpecificRequestor.getClient(
      AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
      selectedPort), new SSLChannelFactory()));

  AvroFlumeEvent avroEvent = new AvroFlumeEvent();

  avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
  avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));

  Status status = client.append(avroEvent);

  Assert.assertEquals(Status.OK, status);

  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "Hello avro ssl",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  logger.debug("Round trip event:{}", event);

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}