org.apache.flume.Sink Java Examples

@Test
public void processOneEventWithBatchSizeOne() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  ctx.put("batchSize", "1");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  assertThat(sink.process()).isEqualTo(Sink.Status.READY);
  verify(table).save(ImmutableList.of(event));
  verifyNoMoreInteractions(table);
  verify(tx).begin();
  verify(tx).commit();
  verify(tx).close();
}
 

@Ignore @Test
public void processOneEvent() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  assertThat(sink.process()).isEqualTo(Sink.Status.READY);
  verify(table).save(ImmutableList.of(event));
  verifyNoMoreInteractions(table);
  verify(tx).begin();
  verify(tx).commit();
  verify(tx).close();
}
 

@Ignore @Test
public void processEmtpyChannel() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  when(channel.take()).thenReturn(null);
  assertThat(sink.process()).isEqualTo(Sink.Status.READY);
  verifyZeroInteractions(table);
}
 

@Override
public void run() {
  run = true;
  while(run && count < until) {
    boolean error = true;
    try {
      if(Sink.Status.READY.equals(sink.process())) {
        count++;
        error = false;
      }
    } catch(Exception ex) {
      errors.add(ex);
    }
    if(error) {
      try {
        Thread.sleep(1000L);
      } catch (InterruptedException e) {}
    }
  }
}
 

private void checkSinkChannelCompatibility(Sink sink, Channel channel)
        throws InstantiationException {
    if (sink instanceof BatchSizeSupported && channel instanceof TransactionCapacitySupported) {
        long transCap = ((TransactionCapacitySupported) channel).getTransactionCapacity();
        long batchSize = ((BatchSizeSupported) sink).getBatchSize();
        if (transCap < batchSize) {
            String msg = String.format(
                    "Incompatible sink and channel settings defined. " +
                            "sink's batch size is greater than the channels transaction capacity. " +
                            "Sink: %s, batch size = %d, channel %s, transaction capacity = %d",
                    sink.getName(), batchSize,
                    channel.getName(), transCap);
            throw new InstantiationException(msg);
        }
    }
}
 

@Test
public void testDuplicateCreate() {


  Sink avroSink1 = sinkFactory.create("avroSink1", "avro");
  Sink avroSink2 = sinkFactory.create("avroSink2", "avro");

  Assert.assertNotNull(avroSink1);
  Assert.assertNotNull(avroSink2);
  Assert.assertNotSame(avroSink1, avroSink2);
  Assert.assertTrue(avroSink1 instanceof AvroSink);
  Assert.assertTrue(avroSink2 instanceof AvroSink);

  Sink s1 = sinkFactory.create("avroSink1", "avro");
  Sink s2 = sinkFactory.create("avroSink2", "avro");

  Assert.assertNotSame(avroSink1, s1);
  Assert.assertNotSame(avroSink2, s2);
}
 

@Test
public void testSinkCreation() {
    SinkFactory factory = new DefaultSinkFactory ();
    Sink sink = factory.create("PhoenixSink__", "com.salesforce.phoenix.flume.sink.PhoenixSink");
    Assert.assertNotNull(sink);
    Assert.assertTrue(PhoenixSink.class.isInstance(sink));
}
 

@Override
public void stop() {
  for(Sink s : sinkList) {
    //s.start();
    s.stop(); //modify by judasheng, FLUME-1718
  }
  state = LifecycleState.STOP;
}
 

@Test
public void testSinkCreation() {
    SinkFactory factory = new DefaultSinkFactory ();
    Sink sink = factory.create("PhoenixSink__", "org.apache.phoenix.flume.sink.PhoenixSink");
    Assert.assertNotNull(sink);
    Assert.assertTrue(PhoenixSink.class.isInstance(sink));
}
 

@Override
public void start() {
  for(Sink s : sinkList) {
    s.start();
  }

  state = LifecycleState.START;
}
 

private Sink moveActiveToDeadAndGetNext() {
  Integer key = liveSinks.lastKey();
  failedSinks.add(new FailedSink(key, activeSink, 1));
  liveSinks.remove(key);
  if(liveSinks.isEmpty()) return null;
  if(liveSinks.lastKey() != null) {
    return liveSinks.get(liveSinks.lastKey());
  } else {
    return null;
  }
}
 

@Override
public Sink create(String name, String type) throws FlumeException {
  Preconditions.checkNotNull(name, "name");
  Preconditions.checkNotNull(type, "type");
  logger.info("Creating instance of sink: {}, type: {}", name, type);
  Class<? extends Sink> sinkClass = getClass(type);
  try {
    Sink sink = sinkClass.newInstance();
    sink.setName(name);
    return sink;
  } catch (Exception ex) {
    throw new FlumeException("Unable to create sink: " + name
        + ", type: " + type + ", class: " + sinkClass.getName(), ex);
  }
}
 

@Override
public void setSinks(List<Sink> sinks) {
  // needed to implement the start/stop functionality
  super.setSinks(sinks);

  this.sinks = new HashMap<String, Sink>();
  for (Sink sink : sinks) {
    this.sinks.put(sink.getName(), sink);
  }
}
 

@Override
public void setSinks(List<Sink> sinks) {
  Preconditions.checkNotNull(sinks);
  Preconditions.checkArgument(sinks.size() == 1, "DefaultSinkPolicy can "
      + "only handle one sink, "
      + "try using a policy that supports multiple sinks");
  sink = sinks.get(0);
}
 

@Test
public void TestOpenAndReadSource() throws Exception {
    Map<String, Object> conf = Maps.newHashMap();
    StringSource stringSource = new StringSource();
    conf.put("name", "a1");
    conf.put("confFile", "./src/test/resources/flume/sink.conf");
    conf.put("noReloadConf", false);
    conf.put("zkConnString", "");
    conf.put("zkBasePath", "");
    Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
    stringSource.open(conf, mockSourceContext);
    Thread.sleep(3 * 1000);
    sink.start();
    Transaction transaction = channel.getTransaction();

    transaction.begin();
    for (int i = 0; i < 10; i++) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();

    for (int i = 0; i < 5; i++) {
        Sink.Status status = sink.process();
        Assert.assertEquals(Sink.Status.READY, status);
    }

    Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
    stringSource.close();
}
 

@Override
public Status process() throws EventDeliveryException {
  Status status = null;

  //if processer is switch off, then just return
  if ( !switchon ) {
  	LOGGER.warn("SinkProcesser is set off, not call sink process.");
  	try {
	Thread.sleep(500);
} catch (InterruptedException e) {
}
  	return Status.READY;
  }
  
  Iterator<Sink> sinkIterator = selector.createSinkIterator();
  while (sinkIterator.hasNext()) {
    Sink sink = sinkIterator.next();
    try {
      status = sink.process();
      break;
    } catch (Exception ex) {
      selector.informSinkFailed(sink);
      LOGGER.warn("Sink failed to consume event. "
          + "Attempting next sink if available.", ex);
    }
  }

  if (status == null) {
    throw new EventDeliveryException("All configured sinks have failed");
  }

  return status;
}
 

@Test
public void testProcess() throws Exception {

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
    port);

  channel.start();
  sink.start();

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 11; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();
  for (int i = 0; i < 6; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertEquals(11, src.flumeEvents.size());
  Assert.assertEquals(6, src.batchCount);
  Assert.assertEquals(0, src.individualCount);

}
 

private LoadBalancingSinkProcessor getProcessor(List<Sink> sinks, Context ctx)
{
  LoadBalancingSinkProcessor lbsp = new LoadBalancingSinkProcessor();
  lbsp.setSinks(sinks);
  lbsp.configure(ctx);
  lbsp.start();

  return lbsp;
}
 

@Test
public void testDefaultConfiguration() throws Exception {
  // If no selector is specified, the round-robin selector should be used
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3*n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor(sinks, new Context());

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == n);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == n);

}
 

@Test
public void testRandomPersistentFailure() throws Exception {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3*n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  // s2 always fails
  s2.setFail(true);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("random",sinks, false);

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s2.getEvents().size() == 0);
  Assert.assertTrue(s1.getEvents().size() + s3.getEvents().size() == 3*n);
}
 

@Test
public void testRoundRobinPersistentFailure() throws Exception {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3*n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  // s2 always fails
  s2.setFail(true);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, false);

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == n);
  Assert.assertTrue(s2.getEvents().size() == 0);
  Assert.assertTrue(s3.getEvents().size() == 2*n);
}
 

@Test
public void testRoundRobinBackoffFailureRecovery() throws EventDeliveryException, InterruptedException {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3*n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);
  s2.setFail(true);

    MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, true);

  Status s = Status.READY;
  for (int i = 0; i < 3 && s != Status.BACKOFF; i++) {
    s = lbsp.process();
  }
  s2.setFail(false);
  Thread.sleep(2001);
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertEquals(n + 1, s1.getEvents().size());
  Assert.assertEquals(n - 1,  s2.getEvents().size());
  Assert.assertEquals(n, s3.getEvents().size());
}
 

@Test
public void testRoundRobinNoFailure() throws Exception {

  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3*n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, false);

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == n);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == n);
}
 

@Test
public void testCustomSelector() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  // This selector will result in all events going to s2
  Context ctx = getContext(FixedOrderSelector.class.getCanonicalName());
  ctx.put("selector." + FixedOrderSelector.SET_ME, "foo");
  LoadBalancingSinkProcessor lbsp = getProcessor(sinks, ctx);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}
 

@Test
public void test() {
  Context context = new Context();
  context.put("type", FailoverSinkProcessor.class.getName());
  context.put("priority.sink1", "1");
  context.put("priority.sink2", "2");
  SinkFactory sf = new DefaultSinkFactory();
  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(sf.create("sink1", "avro"));
  sinks.add(sf.create("sink2", "avro"));
  SinkProcessor sp = SinkProcessorFactory.getProcessor(context, sinks);
  context.put("type", "failover");
  SinkProcessor sp2 = SinkProcessorFactory.getProcessor(context, sinks);
  Assert.assertEquals(sp.getClass(), sp2.getClass());
}
 

@Test
public void testInstantiatingLoadBalancingSinkProcessor() {
  Context context = new Context();
  context.put("type", LoadBalancingSinkProcessor.class.getName());
  context.put("selector", "random");
  SinkFactory sf = new DefaultSinkFactory();
  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(sf.create("sink1", "avro"));
  sinks.add(sf.create("sink2", "avro"));
  SinkProcessor sp = SinkProcessorFactory.getProcessor(context, sinks);
  context.put("type", "load_balance");
  SinkProcessor sp2 = SinkProcessorFactory.getProcessor(context, sinks);
  Assert.assertEquals(sp.getClass(), sp2.getClass());
}
 

@Test
public void testProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

public Sink getSink() {
  return sink;
}
 

@Test
public void testRandomNoFailure() throws Exception {

  Channel ch = new MockChannel();
  int n = 10000;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  MockSink s4 = new MockSink(4);
  s4.setChannel(ch);

  MockSink s5 = new MockSink(5);
  s5.setChannel(ch);

  MockSink s6 = new MockSink(6);
  s6.setChannel(ch);

  MockSink s7 = new MockSink(7);
  s7.setChannel(ch);

  MockSink s8 = new MockSink(8);
  s8.setChannel(ch);

  MockSink s9 = new MockSink(9);
  s9.setChannel(ch);

  MockSink s0 = new MockSink(0);
  s0.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);
  sinks.add(s4);
  sinks.add(s5);
  sinks.add(s6);
  sinks.add(s7);
  sinks.add(s8);
  sinks.add(s9);
  sinks.add(s0);

  LoadBalancingSinkProcessor lbsp = getProcessor("random",sinks, false);

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Set<Integer> sizeSet = new HashSet<Integer>();
  int sum = 0;
  for (Sink ms : sinks) {
    int count = ((MockSink) ms).getEvents().size();
    sum += count;
    sizeSet.add(count);
  }

  // Assert that all the events were accounted for
  Assert.assertEquals(n, sum);

  // Assert that at least two sinks came with different event sizes.
  // This makes sense if the total number of events is evenly divisible by
  // the total number of sinks. In which case the round-robin policy will
  // end up causing all sinks to get the same number of events where as
  // the random policy will have very low probability of doing that.
  Assert.assertTrue("Miraculous distribution", sizeSet.size() > 1);
}
 

@Test
public void testRoundRobinOneActiveSink() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);


  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  // s3 always fails
  s3.setFail(true);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin", sinks, false);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}