org.apache.flume.EventDeliveryException Java Examples

/**
 * {@inheritDoc}
 */
@Override
public Status process() throws EventDeliveryException {
    Status status = Status.READY;

    try {
        Event e = poll();
        if (e != null) {
            getChannelProcessor().processEvent(e);
            status = Status.READY;
        }
    } catch (Throwable t) {
        status = Status.BACKOFF;
        log.error("RestSource error. " + t.getMessage());
    }

    return status;
}
 

/**
 * Lack of exception test.
 */
@Test
public void testAppend() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();
  Configurables.configure(channel, context);
  Configurables.configure(sink, context);

  sink.setChannel(channel);
  sink.start();

  for (int i = 0; i < 10; i++) {
    Event event = EventBuilder.withBody(("Test " + i).getBytes());

    channel.put(event);
    sink.process();
  }

  sink.stop();
}
 

@Test (expected = EventDeliveryException.class)
public void testTimeout() throws Throwable {
  File TESTFILE = new File(TestLoadBalancingLog4jAppender.class
    .getClassLoader()
    .getResource("flume-loadbalancinglog4jtest.properties")
    .getFile());

  ch = new TestLog4jAppender.SlowMemoryChannel(2000);
  configureChannel();
  slowDown = true;
  startSources(TESTFILE, false, new int[]{25430, 25431, 25432});
  int level = 20000;
  String msg = "This is log message number" + String.valueOf(level);
  try {
    fixture.log(Level.toLevel(level), msg);
  } catch (FlumeException ex) {
    throw ex.getCause();
  }

}
 

@Override
public Status process() throws EventDeliveryException {
  synchronized(this) {
    if (remaining <= 0) {
      throw new EventDeliveryException("can't consume more");
    }
  }

  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = channel.take();
  tx.commit();
  tx.close();

  if (e != null) {
    synchronized(this) {
      remaining--;
    }
    written++;
  }

  return Status.READY;
}
 

/**
 * First connect the client, then close the client, then send a request.
 * @throws FlumeException
 * @throws EventDeliveryException
 */
@Test(expected=EventDeliveryException.class)
public void testClientClosedRequest() throws FlumeException,
    EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcTestUtils.getStockLocalClient(server.getPort());
    client.close();
    Assert.assertFalse("Client should not be active", client.isActive());
    System.out.println("Yaya! I am not active after client close!");
    client.append(EventBuilder.withBody("hello", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

private synchronized RpcClient getClient(HostInfo info)
    throws FlumeException, EventDeliveryException {
  throwIfClosed();
  String name = info.getReferenceName();
  RpcClient client = clientMap.get(name);
  if (client == null) {
    client = createClient(name);
    clientMap.put(name, client);
  } else if (!client.isActive()) {
    try {
      client.close();
    } catch (Exception ex) {
      LOGGER.warn("Failed to close client for " + info, ex);
    }
    client = createClient(name);
    clientMap.put(name, client);
  }

  return client;
}
 

@Override
public void append(Event event) throws EventDeliveryException {
  try {
    append(event, requestTimeout, TimeUnit.MILLISECONDS);
  } catch (Throwable t) {
    // we mark as no longer active without trying to clean up resources
    // client is required to call close() to clean up resources
    setState(ConnState.DEAD);
    if (t instanceof Error) {
      throw (Error) t;
    }
    if (t instanceof TimeoutException) {
      throw new EventDeliveryException(this + ": Failed to send event. " +
          "RPC request timed out after " + requestTimeout + "ms", t);
    }
    throw new EventDeliveryException(this + ": Failed to send event", t);
  }
}
 

@Override
public void processEvents(List<Event> events) {
	int batchSize = 10;
	int batches = events.size() / batchSize;
	for (int i = 0; i <= batches; i++) {
		Transaction txn = channel.getTransaction();
		txn.begin();
		int from = batchSize * i;
		int to = (i == batches) ? events.size() : from + batchSize;
		for (Event event : events.subList(from, to)) {
			channel.put(event);
			LOG.debug("Putting event to channel: {}", event);
		}
		txn.commit();
		txn.close();
		try {
			sink.process();
		} catch (EventDeliveryException e) {
			LOG.error("Error processing events!", e);
			throw new RuntimeException("Error processing events!", e);
		}
	}
}
 

private void handleTransactionFailure(Transaction txn)
    throws EventDeliveryException {
  try {
    txn.rollback();
  } catch (Throwable e) {
    logger.error("Failed to commit transaction." +
        "Transaction rolled back.", e);
    if(e instanceof Error || e instanceof RuntimeException){
      logger.error("Failed to commit transaction." +
          "Transaction rolled back.", e);
      Throwables.propagate(e);
    } else {
      logger.error("Failed to commit transaction." +
          "Transaction rolled back.", e);
      throw new EventDeliveryException("Failed to commit transaction." +
          "Transaction rolled back.", e);
    }
  } finally {
    txn.close();
  }
}
 

@Override
public void processEvents(List<Event> events) {
	int batchSize = 10;
	int batches = events.size() / batchSize;
	for (int i = 0; i <= batches; i++) {
		Transaction txn = channel.getTransaction();
		txn.begin();
		int from = batchSize * i;
		int to = (i == batches) ? events.size() : from + batchSize;
		for (Event event : events.subList(from, to)) {
			channel.put(event);
			LOG.debug("Putting event to channel: {}", event);
		}
		txn.commit();
		txn.close();
		try {
			sink.process();
		} catch (EventDeliveryException e) {
			LOG.error("Error processing events!", e);
			throw new RuntimeException("Error processing events!", e);
		}
	}
}
 

@Test
public void testTwoParamBatchAppendOverflow() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort());
    int batchSize = client.getBatchSize();
    int moreThanBatch = batchSize + 1;
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < moreThanBatch; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

@Test
public void testPropertiesBatchAppend() throws FlumeException,
    EventDeliveryException {
  int batchSize = 7;
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    Properties p = new Properties();
    p.put("hosts", "host1");
    p.put("hosts.host1", localhost + ":" + String.valueOf(server.getPort()));
    p.put("batch-size", String.valueOf(batchSize));
    client = RpcClientFactory.getInstance(p);
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < batchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

@Test
public void testThreeParamBatchAppend() throws FlumeException,
    EventDeliveryException {
  int batchSize = 7;
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort(),
        batchSize);

    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < batchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

private void send(int numberOfMsgs) throws EventDeliveryException {
  for (int count = 0; count < numberOfMsgs; count++) {
    int level = count % 5;
    String msg = "This is log message number" + String.valueOf(count);
    fixture.log(Level.toLevel(level), msg);
  }
}
 

@Override
public Status process() throws EventDeliveryException {
    Channel channel = getChannel();
    Transaction tx = channel.getTransaction();
    try {
        tx.begin();
        Event event = channel.take();
        if (event == null) {
            tx.commit();
            return Status.READY;

        }

        String data = null;
        if(writeBody){
            data = new String(event.getBody());
        } else {
            data = mapper.writeValueAsString(event.getHeaders());
        }

        producer.send(new KeyedMessage<String, String>(topic, data));
        tx.commit();
        return Status.READY;
    } catch (Exception e) {
        try {
            tx.rollback();
            return Status.BACKOFF;
        } catch (Exception e2) {
            log.error("Rollback Exception:{}", e2);
        }
        log.error("KafkaSink Exception:{}", e);
        return Status.BACKOFF;
    } finally {
        tx.close();
    }
}
 

@Test
public void testRoundRobinBackoffFailureRecovery() throws EventDeliveryException, InterruptedException {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3*n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);
  s2.setFail(true);

    MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, true);

  Status s = Status.READY;
  for (int i = 0; i < 3 && s != Status.BACKOFF; i++) {
    s = lbsp.process();
  }
  s2.setFail(false);
  Thread.sleep(2001);
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertEquals(n + 1, s1.getEvents().size());
  Assert.assertEquals(n - 1,  s2.getEvents().size());
  Assert.assertEquals(n, s3.getEvents().size());
}
 

/**
 * Helper method for testing batch appends on handlers
 * @param handler
 * @throws FlumeException
 * @throws EventDeliveryException
 */
public static void handlerBatchAppendTest(AvroSourceProtocol handler, boolean enableServerCompression, boolean enableClientCompression, int compressionLevel)
    throws FlumeException, EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = startServer(handler, 0 , enableServerCompression);
  try {

    Properties starterProp = new Properties();
    if (enableClientCompression) {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL, "" + compressionLevel);
    } else {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
    }

    client = getStockLocalClient(server.getPort(), starterProp);
    boolean isActive = client.isActive();
    Assert.assertTrue("Client should be active", isActive);

    int batchSize = client.getBatchSize();
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < batchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);

  } finally {
    stopServer(server);
    if (client != null) client.close();
  }
}
 

/**
 * Send an event to an online server that returns FAILED.
 */
@Test(expected=EventDeliveryException.class)
public void testFailedServerSimple() throws FlumeException,
    EventDeliveryException {

  RpcTestUtils.handlerSimpleAppendTest(new FailedAvroHandler());
  logger.error("Failed: I should never have gotten here!");
}
 

@Test
public void testLifecycle() throws InterruptedException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();

  for (long i = 0; i < 100; i++) {
    source.process();
    Event event = channel.take();

    Assert.assertArrayEquals(String.valueOf(i).getBytes(),
        new String(event.getBody()).getBytes());
  }
  source.stop();
}
 

@SuppressWarnings("unchecked")
@Test
public void testProcessStatusReady() throws EventDeliveryException, SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException {
	when(mockEvent.getBody()).thenReturn("frank".getBytes());
	Status status = mockKafkaSink.process();
	verify(mockChannel, times(1)).getTransaction();
	verify(mockChannel, times(1)).take();
	verify(mockProducer, times(1)).send((ProducerData<String, String>) any());
	verify(mockTx, times(1)).commit();
	verify(mockTx, times(0)).rollback();
	verify(mockTx, times(1)).close();
	assertEquals(Status.READY, status);
}
 

@Test
public void testTwoParamDeprecatedAppend() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getInstance(localhost, server.getPort());
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

@Test
public void testTwoParamSimpleAppend() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort());
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

@Test(expected = EventDeliveryException.class)
public void testTwoHostFailoverThrowAfterClose() throws Exception {
  Server s1 = null, s2 = null;
  RpcClient c = null;
  try{
    LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
    LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();

    s1 = RpcTestUtils.startServer(h1);
    s2 = RpcTestUtils.startServer(h2);

    Properties p = new Properties();
    p.put("hosts", "h1 h2");
    p.put("client.type", "default_loadbalance");
    p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
    p.put("hosts.h2", "127.0.0.1:" + s2.getPort());

    c = RpcClientFactory.getInstance(p);
    Assert.assertTrue(c instanceof LoadBalancingRpcClient);

    for (int i = 0; i < 100; i++) {
      if (i == 20) {
        h2.setFailed();
      } else if (i == 40) {
        h2.setOK();
      }
      c.append(getEvent(i));
    }

    Assert.assertEquals(60, h1.getAppendCount());
    Assert.assertEquals(40, h2.getAppendCount());
    if (c != null) c.close();
    c.append(getEvent(3));
    Assert.fail();
  } finally {
    if (s1 != null) s1.close();
    if (s2 != null) s2.close();
  }
}
 

@SuppressWarnings("unchecked")
@Test
public void testProcessItEmpty() throws EventDeliveryException, SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException {
	when(mockIt.next()).thenReturn(mockMessageAndMetadata);
	when(mockIt.hasNext()).thenReturn(false);
	Status status = mockKafkaSource.process();
	verify(mockIt, times(1)).hasNext();
	verify(mockIt, times(0)).next();
	verify(mockChannelProcessor, times(1)).processEventBatch(anyList());
	when(mockIt.next()).thenReturn(mockMessageAndMetadata);
	assertEquals(Status.READY, status);
}
 

/**
 * Not thread-safe.
 *
 * @param event
 * @param reuse
 * @return
 */
private GenericRecord deserialize(Event event, GenericRecord reuse)
    throws EventDeliveryException {
  decoder = DecoderFactory.get().binaryDecoder(event.getBody(), decoder);
  // no checked exception is thrown in the CacheLoader
  DatumReader<GenericRecord> reader = readers.getUnchecked(schema(event));
  try {
    return reader.read(reuse, decoder);
  } catch (IOException ex) {
    throw new EventDeliveryException("Cannot deserialize event", ex);
  }
}
 

@Test
public void testProcessNoStart() throws Exception {
  try {
    source.process();
    Assert.fail();
  } catch (EventDeliveryException expected) {

  }
}
 

private void send(int numberOfMsgs) throws EventDeliveryException {
  for (int count = 0; count < numberOfMsgs; count++) {
    int level = count % 5;
    String msg = "This is log message number" + String.valueOf(count);
    fixture.log(Level.toLevel(level), msg);
  }
}
 

@Test
public void testRandomBackoff() throws Exception {
  File TESTFILE = new File(TestLoadBalancingLog4jAppender.class
      .getClassLoader()
      .getResource("flume-loadbalancing-backoff-log4jtest.properties")
      .getFile());
  startSources(TESTFILE, new int[] { 25430, 25431, 25432 });

  sources.get(0).setFail();
  sources.get(2).setFail();

  sendAndAssertMessages(50);

  Assert.assertEquals(50, sources.get(1).appendCount.intValue());
  Assert.assertEquals(0, sources.get(0).appendCount.intValue());
  Assert.assertEquals(0, sources.get(2).appendCount.intValue());
  sources.get(0).setOk();
  sources.get(1).setFail(); // s0 should still be backed off
  try {
    send(1);
    // nothing should be able to process right now
    Assert.fail("Expected EventDeliveryException");
  } catch (FlumeException e) {
    Assert.assertTrue(e.getCause() instanceof EventDeliveryException);
  }
  Thread.sleep(2500); // wait for s0 to no longer be backed off

  sendAndAssertMessages(50);

  Assert.assertEquals(50, sources.get(0).appendCount.intValue());
  Assert.assertEquals(50, sources.get(1).appendCount.intValue());
  Assert.assertEquals(0, sources.get(2).appendCount.intValue());
}
 

@Test
public void testGetFlumeAgent() throws EventDeliveryException,
		InterruptedException {

	EmbeddedAgent flumeAgent = flumeAgentService.getFlumeAgent();

	List<Event> searchEvents = generateSearchAnalyticsDataService
			.getSearchEvents(searchEventsCount);
	for (Event event : searchEvents) {
		flumeAgent.put(event);
	}
	// wait until sink process everything. sleep 10 sec.
	Thread.sleep(10000);
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    if (source instanceof PollableSource) {
        PollableSource pollableSource = (PollableSource) source;
        try {
            pollableSourceChannel.setSession(session);
            pollableSource.process();
        } catch (EventDeliveryException ex) {
            throw new ProcessException("Error processing pollable source", ex);
        }
    } else {
        throw new ProcessException("Invalid source type: " + source.getClass().getSimpleName());
    }
}