Java Code Examples for org.apache.flume.Sink#Status

@Test
public void TestOpenAndReadSource() throws Exception {
    Map<String, Object> conf = Maps.newHashMap();
    StringSource stringSource = new StringSource();
    conf.put("name", "a1");
    conf.put("confFile", "./src/test/resources/flume/sink.conf");
    conf.put("noReloadConf", false);
    conf.put("zkConnString", "");
    conf.put("zkBasePath", "");
    Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
    stringSource.open(conf, mockSourceContext);
    Thread.sleep(3 * 1000);
    sink.start();
    Transaction transaction = channel.getTransaction();

    transaction.begin();
    for (int i = 0; i < 10; i++) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();

    for (int i = 0; i < 5; i++) {
        Sink.Status status = sink.process();
        Assert.assertEquals(Sink.Status.READY, status);
    }

    Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
    stringSource.close();
}
 

@Test
public void testProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

@Test
public void testProcess() throws Exception {

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
    port);

  channel.start();
  sink.start();

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 11; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();
  for (int i = 0; i < 6; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertEquals(11, src.flumeEvents.size());
  Assert.assertEquals(6, src.batchCount);
  Assert.assertEquals(0, src.individualCount);

}
 

@Test
public void testCustomSelector() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  // This selector will result in all events going to s2
  Context ctx = getContext(FixedOrderSelector.class.getCanonicalName());
  ctx.put("selector." + FixedOrderSelector.SET_ME, "foo");
  LoadBalancingSinkProcessor lbsp = getProcessor(sinks, ctx);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}
 

@Test
public void testFailedConnect() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {

  setUp();
  Event event = EventBuilder.withBody("test event 1",
      Charset.forName("UTF8"));
  Server server = createServer(new MockAvroServer());

  server.start();
  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Thread.sleep(500L); // let socket startup
  server.close();
  Thread.sleep(500L); // sleep a little to allow close occur

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    boolean threwException = false;
    try {
      sink.process();
    } catch (EventDeliveryException e) {
      threwException = true;
    }
    Assert.assertTrue("Must throw EventDeliveryException if disconnected",
        threwException);
  }

  server = createServer(new MockAvroServer());
  server.start();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));
  server.close();
}
 

@Test
public void testSslProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

@Test
public void testSslProcessWithTrustStore() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("truststore", "src/test/resources/truststore.jks");
  context.put("truststore-password", "password");
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

@Test
public void testFailedConnect() throws Exception {

  Event event = EventBuilder.withBody("test event 1",
    Charset.forName("UTF8"));

  sink.start();

  Thread.sleep(500L); // let socket startup
  Thread.sleep(500L); // sleep a little to allow close occur

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    boolean threwException = false;
    try {
      sink.process();
    } catch (EventDeliveryException e) {
      threwException = true;
    }
    Assert.assertTrue("Must throw EventDeliveryException if disconnected",
      threwException);
  }

  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
    port);

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
  sink.stop();
}
 

@Test
public void testRandomOneActiveSink() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);


  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  // s3 always fails
  s3.setFail(true);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("random", sinks, false);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}
 

@Test
public void testRandomBackoff() throws Exception {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  // s3 always fails
  s3.setFail(true);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("random", sinks, true);

  // TODO: there is a remote possibility that s0 or s2
  // never get hit by the random assignment
  // and thus not backoffed, causing the test to fail
  for(int i=0; i < 50; i++) {
    // a well behaved runner would always check the return.
    lbsp.process();
  }
  Assert.assertEquals(50, s2.getEvents().size());
  s2.setFail(true);
  s1.setFail(false); // s1 should still be backed off
  try {
    lbsp.process();
    // nothing should be able to process right now
    Assert.fail("Expected EventDeliveryException");
  } catch (EventDeliveryException e) {
    // this is expected
  }
  Thread.sleep(2100); // wait for s1 to no longer be backed off
  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertEquals(50, s1.getEvents().size());
  Assert.assertEquals(50, s2.getEvents().size());
  Assert.assertEquals(0, s3.getEvents().size());
}
 

@Test
public void testRoundRobinOneActiveSink() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);


  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  // s3 always fails
  s3.setFail(true);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin", sinks, false);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}