org.apache.flume.source.avro.AvroSourceProtocol Java Examples

public static Server startTestFlumeServer(int port) {
  Responder responder = new SpecificResponder(AvroSourceProtocol.class,
          new OKAvroHandler());
  Server server = new NettyServer(responder,
            new InetSocketAddress("127.0.0.1", port));

  server.start();
  LOG.info("Server started on test flume server hostname: localhost, port: " + port);

  try {

    Thread.sleep(1000L);

  } catch (InterruptedException ex) {
    LOG.error("Thread interrupted. Exception follows.", ex);
    Thread.currentThread().interrupt();
  }

  return server;
}
 

/**
 * Helper method for testing simple (single) with compression level 6 appends on handlers
 * @param handler
 * @throws FlumeException
 * @throws EventDeliveryException
 */
public static void handlerSimpleAppendTest(AvroSourceProtocol handler, boolean enableServerCompression, boolean enableClientCompression, int compressionLevel)
    throws FlumeException, EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = startServer(handler, 0, enableServerCompression);
  try {
    Properties starterProp = new Properties();
    if (enableClientCompression) {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL, "" + compressionLevel);
    } else {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
    }
    client = getStockLocalClient(server.getPort(), starterProp);
    boolean isActive = client.isActive();
    Assert.assertTrue("Client should be active", isActive);
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    stopServer(server);
    if (client != null) client.close();
  }
}
 

private void start() {
  flumeServer = RpcTestUtils.startServer(new AvroSourceProtocol(){

    @Override
    public Status append(AvroFlumeEvent event) throws AvroRemoteException {
      return protocol.append(event);
    }

    @Override
    public Status appendBatch(List<AvroFlumeEvent> events) throws AvroRemoteException {
      return protocol.appendBatch(events);
    }
  });
}
 

/**
 * Helper method for testing batch appends on handlers
 * @param handler
 * @throws FlumeException
 * @throws EventDeliveryException
 */
public static void handlerBatchAppendTest(AvroSourceProtocol handler, boolean enableServerCompression, boolean enableClientCompression, int compressionLevel)
    throws FlumeException, EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = startServer(handler, 0 , enableServerCompression);
  try {

    Properties starterProp = new Properties();
    if (enableClientCompression) {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL, "" + compressionLevel);
    } else {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
    }

    client = getStockLocalClient(server.getPort(), starterProp);
    boolean isActive = client.isActive();
    Assert.assertTrue("Client should be active", isActive);

    int batchSize = client.getBatchSize();
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < batchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);

  } finally {
    stopServer(server);
    if (client != null) client.close();
  }
}
 

@Before
public void setUp() throws Exception {
  headers = Maps.newHashMap();
  headers.put("key1", "value1");
  body = "body".getBytes(Charsets.UTF_8);

  int port = findFreePort();
  eventCollector = new EventCollector();
  Responder responder = new SpecificResponder(AvroSourceProtocol.class,
      eventCollector);
  nettyServer = new NettyServer(responder,
            new InetSocketAddress(HOSTNAME, port));
  nettyServer.start();

  // give the server a second to start
  Thread.sleep(1000L);

  properties = Maps.newHashMap();
  properties.put("channel.type", "memory");
  properties.put("channel.capacity", "200");
  properties.put("sinks", "sink1 sink2");
  properties.put("sink1.type", "avro");
  properties.put("sink2.type", "avro");
  properties.put("sink1.hostname", HOSTNAME);
  properties.put("sink1.port", String.valueOf(port));
  properties.put("sink2.hostname", HOSTNAME);
  properties.put("sink2.port", String.valueOf(port));
  properties.put("processor.type", "load_balance");

  agent = new EmbeddedAgent("test-" + serialNumber.incrementAndGet());
}
 

@Override
public void start() {
  logger.info("Starting {}...", this);

  Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);

  NioServerSocketChannelFactory socketChannelFactory = initSocketChannelFactory();

  ChannelPipelineFactory pipelineFactory = initChannelPipelineFactory();

  server = new NettyServer(responder, new InetSocketAddress(bindAddress, port),
        socketChannelFactory, pipelineFactory, null);

  connectionCountUpdater = Executors.newSingleThreadScheduledExecutor();
  server.start();
  sourceCounter.start();
  super.start();
  final NettyServer srv = (NettyServer)server;
  connectionCountUpdater.scheduleWithFixedDelay(new Runnable(){

    @Override
    public void run() {
      sourceCounter.setOpenConnectionCount(
              Long.valueOf(srv.getNumActiveConnections()));
    }
  }, 0, 60, TimeUnit.SECONDS);

  logger.info("Avro source {} started.", getName());
}
 

private Server createServer(AvroSourceProtocol protocol)
    throws IllegalAccessException, InstantiationException {

  Server server = new NettyServer(new SpecificResponder(
      AvroSourceProtocol.class, protocol), new InetSocketAddress(
      hostname, port));

  return server;
}
 

private Server createSslServer(AvroSourceProtocol protocol)
    throws IllegalAccessException, InstantiationException {
  Server server = new NettyServer(new SpecificResponder(
      AvroSourceProtocol.class, protocol), new InetSocketAddress(hostname, port),
          new NioServerSocketChannelFactory(
                  Executors.newCachedThreadPool(),
                  Executors.newCachedThreadPool()),
          new SSLChannelPipelineFactory(),
          null);

  return server;
}
 

public EventCollector(final int port) {
    final Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);
    nettyServer = new NettyServer(responder, new InetSocketAddress(HOSTNAME, port));
    nettyServer.start();
}
 

public EventCollector(final int port) {
    final Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);
    System.out.println("Collector listening on port " + port);
    nettyServer = new NettyServer(responder, new InetSocketAddress(HOSTNAME, port));
    nettyServer.start();
}
 

public EventCollector(final int port) {
    final Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);
    nettyServer = new NettyServer(responder, new InetSocketAddress(HOSTNAME, port));
    nettyServer.start();
}
 

public EventCollector(final int port) {
    final Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);
    nettyServer = new NettyServer(responder, new InetSocketAddress(HOSTNAME, port));
    nettyServer.start();
}
 

@Test
public void testSslRequest() throws InterruptedException, IOException {
  boolean bound = false;

  for (int i = 0; i < 10 && !bound; i++) {
    try {
      Context context = new Context();

      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");
      context.put("ssl", "true");
      context.put("keystore", "src/test/resources/server.p12");
      context.put("keystore-password", "password");
      context.put("keystore-type", "PKCS12");

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
      Thread.sleep(100);
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  AvroSourceProtocol client = SpecificRequestor.getClient(
      AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
      selectedPort), new SSLChannelFactory()));

  AvroFlumeEvent avroEvent = new AvroFlumeEvent();

  avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
  avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));

  Status status = client.append(avroEvent);

  Assert.assertEquals(Status.OK, status);

  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "Hello avro ssl",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  logger.debug("Round trip event:{}", event);

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

private void doRequest(boolean serverEnableCompression, boolean clientEnableCompression, int compressionLevel) throws InterruptedException, IOException {
  boolean bound = false;

  for (int i = 0; i < 100 && !bound; i++) {
    try {
      Context context = new Context();
      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");
      context.put("threads", "50");
      if (serverEnableCompression) {
        context.put("compression-type", "deflate");
      } else {
        context.put("compression-type", "none");
      }

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  AvroSourceProtocol client;
  if (clientEnableCompression) {
    client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort), new CompressionChannelFactory(6)));
  } else {
    client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort)));
  }

  AvroFlumeEvent avroEvent = new AvroFlumeEvent();

  avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
  avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));

  Status status = client.append(avroEvent);

  Assert.assertEquals(Status.OK, status);

  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "Hello avro",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  logger.debug("Round trip event:{}", event);

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

public static Server startServer(AvroSourceProtocol handler, int port) {
  return startServer(handler, port, false);
}
 

public static Server startServer(AvroSourceProtocol handler) {
  return startServer(handler, 0, false);
}
 

public static void handlerBatchAppendTest(AvroSourceProtocol handler)
    throws FlumeException, EventDeliveryException {
  handlerBatchAppendTest(handler, false, false, 0);
}
 

@Test
public void flumeAvroIpc() throws StageException, IOException, ExecutionException, InterruptedException {

  final Charset charset = Charsets.UTF_8;
  final TCPServerSourceConfig configBean = createConfigBean(charset);
  configBean.tcpMode = TCPMode.FLUME_AVRO_IPC;
  configBean.dataFormat = DataFormat.TEXT;
  configBean.bindAddress = "0.0.0.0";

  final int batchSize = 5;
  final String outputLane = "output";

  final TCPServerSource source = new TCPServerSource(configBean);
  final PushSourceRunner runner = new PushSourceRunner.Builder(TCPServerDSource.class, source)
      .addOutputLane(outputLane)
      .setOnRecordError(OnRecordError.TO_ERROR)
      .build();

  runner.runInit();

  runner.runProduce(Collections.emptyMap(), batchSize, out -> {
    final Map<String, List<Record>> outputMap = out.getRecords();
    assertThat(outputMap, hasKey(outputLane));
    final List<Record> records = outputMap.get(outputLane);
    assertThat(records, hasSize(batchSize));
    for (int i = 0; i < batchSize; i++) {
      assertThat(
          records.get(i).get("/" + TextDataParserFactory.TEXT_FIELD_NAME),
          fieldWithValue(getFlumeAvroIpcEventName(i))
      );
    }
    runner.setStop();
  });

  final AvroSourceProtocol client = SpecificRequestor.getClient(AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress("localhost", Integer.parseInt(configBean.ports.get(0)))));

  List<AvroFlumeEvent> events = new LinkedList<>();
  for (int i = 0; i < batchSize; i++) {
    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
    avroEvent.setBody(ByteBuffer.wrap(getFlumeAvroIpcEventName(i).getBytes()));
    events.add(avroEvent);
  }

  Status status = client.appendBatch(events);

  assertThat(status, equalTo(Status.OK));

  runner.waitOnProduce();
}
 

@Override
protected List<ConfigIssue> init() {
  List<ConfigIssue> issues = new ArrayList<>();

  config.dataFormatConfig.stringBuilderPoolSize = config.numThreads;

  if (config.enableEpoll && !Epoll.isAvailable()) {
    issues.add(getContext().createConfigIssue(Groups.TCP.name(), CONF_PREFIX + "enableEpoll", Errors.TCP_05));
  }
  final String portsField = "ports";
  if (config.ports.isEmpty()) {
    issues.add(getContext().createConfigIssue(Groups.TCP.name(), portsField, Errors.TCP_02));
  } else {
    for (String candidatePort : config.ports) {
      try {
        int port = Integer.parseInt(candidatePort.trim());
        if (port > 0 && port < 65536) {
          if (port < 1024) {
            privilegedPortUsage = true; // only for error handling purposes
          }
          addresses.add(new InetSocketAddress(port));
        } else {
          issues.add(getContext().createConfigIssue(Groups.TCP.name(), portsField, Errors.TCP_03, port));
        }
      } catch (NumberFormatException ex) {
        issues.add(getContext().createConfigIssue(Groups.TCP.name(), portsField, Errors.TCP_03, candidatePort));
      }
    }
  }

  if (issues.isEmpty()) {
    if (addresses.isEmpty()) {
      issues.add(getContext().createConfigIssue(Groups.TCP.name(), portsField, Errors.TCP_09));
    } else {
      if (config.tlsConfigBean.isEnabled()) {
        boolean tlsValid = config.tlsConfigBean.init(
            getContext(),
            Groups.TLS.name(),
            CONF_PREFIX + "tlsConfigBean.",
            issues
        );
        if (!tlsValid) {
          return issues;
        }
      }

      final boolean elValid = validateEls(issues);
      if (!elValid) {
        return issues;
      }

      validateTcpConfigs(issues);
      if (issues.size() > 0) {
        return issues;
      }

      if (config.tcpMode == TCPMode.FLUME_AVRO_IPC) {
        config.dataFormatConfig.init(
            getContext(),
            config.dataFormat,
            Groups.TCP.name(),
            CONF_PREFIX + "dataFormatConfig",
            config.maxMessageSize,
            issues
        );
        parserFactory = config.dataFormatConfig.getParserFactory();

        final int avroIpcPort = Integer.parseInt(config.ports.get(0));
        final SpecificResponder avroIpcResponder = new SpecificResponder(
            AvroSourceProtocol.class,
            new SDCFlumeAvroIpcProtocolHandler(getContext(), parserFactory, pipelineIdsToFail::put)
        );

        // this uses Netty 3.x code to help avoid rewriting a lot in our own stage lib
        // Netty 3.x and 4.x (which we use for the other modes) can coexist on the same classpath, so should be OK
        avroIpcServer = new NettyServer(
            avroIpcResponder,
            new InetSocketAddress(config.bindAddress, avroIpcPort)
        );

        avroIpcServer.start();
      } else {
        createAndStartTCPServer(issues, portsField);
      }
    }
  }
  return issues;
}
 

/**
 * Helper method for testing simple (single) appends on handlers
 * @param handler
 * @throws FlumeException
 * @throws EventDeliveryException
 */
public static void handlerSimpleAppendTest(AvroSourceProtocol handler)
    throws FlumeException, EventDeliveryException {
  handlerSimpleAppendTest(handler, false, false, 0);
}