org.apache.flume.source.avro.Status Java Examples

@Override
public Status append(AvroFlumeEvent avroEvent) {
  logger.debug("Avro source {}: Received avro event: {}", getName(),
      avroEvent);
  sourceCounter.incrementAppendReceivedCount();
  sourceCounter.incrementEventReceivedCount();

  Event event = EventBuilder.withBody(avroEvent.getBody().array(),
      toStringMap(avroEvent.getHeaders()));

  try {
    getChannelProcessor().processEvent(event);
  } catch (ChannelException ex) {
    logger.warn("Avro source " + getName() + ": Unable to process event. " +
        "Exception follows.", ex);
    return Status.FAILED;
  }

  sourceCounter.incrementAppendAcceptedCount();
  sourceCounter.incrementEventAcceptedCount();

  return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) {
  if (isFail) {
    return Status.FAILED;
  }
  appendCount.addAndGet(events.size());
  return super.appendBatch(events);
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events)
    throws AvroRemoteException {
  logger.debug("Received event batch:{}; delaying for {}ms", events, delay);
  sleep();
  return Status.OK;
}
 

@Override
public Status append(AvroFlumeEvent avroEvent) {
  if (isFail) {
    return Status.FAILED;
  }
  appendCount.incrementAndGet();
  return super.append(avroEvent);
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) {
  if (isFail) {
    return Status.FAILED;
  }
  appendCount.addAndGet(events.size());
  return super.appendBatch(events);
}
 

@Test
public void testProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) {
  logger.debug("Avro source {}: Received avro event batch of {} events.",
      getName(), events.size());
  sourceCounter.incrementAppendBatchReceivedCount();
  sourceCounter.addToEventReceivedCount(events.size());

  List<Event> batch = new ArrayList<Event>();

  for (AvroFlumeEvent avroEvent : events) {
    Event event = EventBuilder.withBody(avroEvent.getBody().array(),
        toStringMap(avroEvent.getHeaders()));

    batch.add(event);
  }

  try {
    getChannelProcessor().processEventBatch(batch);
  } catch (Throwable t) {
    logger.error("Avro source " + getName() + ": Unable to process event " +
        "batch. Exception follows.", t);
    if (t instanceof Error) {
      throw (Error) t;
    }
    return Status.FAILED;
  }

  sourceCounter.incrementAppendBatchAcceptedCount();
  sourceCounter.addToEventAcceptedCount(events.size());

  return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  logger.info("Throwing: Received {} events from appendBatch()",
      events.size());
  throw new AvroRemoteException("Handler smash!");
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  logger.info("Unknown: Received {} events from appendBatch()",
      events.size());
  return Status.UNKNOWN;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  logger.info("Failed: Received {} events from appendBatch()",
      events.size());
  return Status.FAILED;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  logger.info("OK: Received {} events from appendBatch()",
      events.size());
  return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  if (failed) {
    logger.debug("Event batch rejected");
    return Status.FAILED;
  }
  logger.debug("LB: Received {} events from appendBatch()",
      events.size());

  appendBatchCount++;
  return Status.OK;
}
 

@Override
public Status append(AvroFlumeEvent event) throws AvroRemoteException {
  if (failed) {
    logger.debug("Event rejected");
    return Status.FAILED;
  }
  logger.debug("LB: Received event from append(): {}",
      new String(event.getBody().array(), Charset.forName("UTF8")));
  appendCount++;
  return Status.OK;
}
 

private void start() {
  flumeServer = RpcTestUtils.startServer(new AvroSourceProtocol(){

    @Override
    public Status append(AvroFlumeEvent event) throws AvroRemoteException {
      return protocol.append(event);
    }

    @Override
    public Status appendBatch(List<AvroFlumeEvent> events) throws AvroRemoteException {
      return protocol.appendBatch(events);
    }
  });
}
 

@Override
public Status append(AvroFlumeEvent avroEvent) {
  if (isFail) {
    return Status.FAILED;
  }
  appendCount.incrementAndGet();
  return super.append(avroEvent);
}
 

@Override
public Status appendBatch(final List<AvroFlumeEvent> events) throws AvroRemoteException {
    Preconditions.checkState(eventQueue.addAll(events));
    for (final AvroFlumeEvent event : events) {
        // System.out.println("Received event " + event.getHeaders().get(new org.apache.avro.util.Utf8(FlumeEvent.GUID)));
    }
    return Status.OK;
}
 

@Override
public Status append(final AvroFlumeEvent event) throws AvroRemoteException {
    eventQueue.add(event);
    //System.out.println("Received event " + event.getHeaders().get(new org.apache.avro.util.Utf8(FlumeEvent.GUID)));
    return Status.OK;
}
 

@Test
public void testSslProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

private void doRequest(boolean serverEnableCompression, boolean clientEnableCompression, int compressionLevel) throws InterruptedException, IOException {
  boolean bound = false;

  for (int i = 0; i < 100 && !bound; i++) {
    try {
      Context context = new Context();
      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");
      context.put("threads", "50");
      if (serverEnableCompression) {
        context.put("compression-type", "deflate");
      } else {
        context.put("compression-type", "none");
      }

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  AvroSourceProtocol client;
  if (clientEnableCompression) {
    client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort), new CompressionChannelFactory(6)));
  } else {
    client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort)));
  }

  AvroFlumeEvent avroEvent = new AvroFlumeEvent();

  avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
  avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));

  Status status = client.append(avroEvent);

  Assert.assertEquals(Status.OK, status);

  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "Hello avro",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  logger.debug("Round trip event:{}", event);

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

@Test
public void testSslRequest() throws InterruptedException, IOException {
  boolean bound = false;

  for (int i = 0; i < 10 && !bound; i++) {
    try {
      Context context = new Context();

      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");
      context.put("ssl", "true");
      context.put("keystore", "src/test/resources/server.p12");
      context.put("keystore-password", "password");
      context.put("keystore-type", "PKCS12");

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
      Thread.sleep(100);
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  AvroSourceProtocol client = SpecificRequestor.getClient(
      AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
      selectedPort), new SSLChannelFactory()));

  AvroFlumeEvent avroEvent = new AvroFlumeEvent();

  avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
  avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));

  Status status = client.append(avroEvent);

  Assert.assertEquals(Status.OK, status);

  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "Hello avro ssl",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  logger.debug("Round trip event:{}", event);

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

@Override
public Status append(AvroFlumeEvent event) throws AvroRemoteException {
  eventQueue.add(event);
  return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events)
    throws AvroRemoteException {
  Preconditions.checkState(eventQueue.addAll(events));
  return Status.OK;
}
 

@Override
public Status appendBatch(final List<AvroFlumeEvent> events)
    throws AvroRemoteException {
    Preconditions.checkState(eventQueue.addAll(events));
    return Status.OK;
}
 

@Override
public Status appendBatch(final List<AvroFlumeEvent> events)
    throws AvroRemoteException {
    Preconditions.checkState(eventQueue.addAll(events));
    return Status.OK;
}
 

@Override
public Status append(final AvroFlumeEvent event) throws AvroRemoteException {
    eventQueue.add(event);
    return Status.OK;
}
 

@Override
public Status append(final AvroFlumeEvent event) throws AvroRemoteException {
    eventQueue.add(event);
    return Status.OK;
}
 

@Override
public Status appendBatch(final List<AvroFlumeEvent> events)
    throws AvroRemoteException {
    Preconditions.checkState(eventQueue.addAll(events));
    return Status.OK;
}
 

@Override
public Status append(final AvroFlumeEvent event) throws AvroRemoteException {
    eventQueue.add(event);
    return Status.OK;
}
 

@Override
public Status append(AvroFlumeEvent event) throws AvroRemoteException {
  logger.debug("Received event:{}; delaying for {}ms", event, delay);
  sleep();
  return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events)
    throws AvroRemoteException {
  logger.debug("Received event batch:{}", events);
  return Status.OK;
}