org.apache.flume.Transaction Java Examples

private void sendAndAssertFail(Logger logger) throws Throwable {
    /*
     * Log4j internally defines levels as multiples of 10000. So if we
     * create levels directly using count, the level will be set as the
     * default.
     */
  int level = 20000;
  try {
    logger.log(Level.toLevel(level), "Test Msg");
  } catch (FlumeException ex) {
    ex.printStackTrace();
    throw ex.getCause();
  }
  Transaction transaction = ch.getTransaction();
  transaction.begin();
  Event event = ch.take();
  Assert.assertNull(event);
  transaction.commit();
  transaction.close();

}
 

@Test
public void testTake2() throws Exception {
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.take();
  transaction.rollback();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.take();
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.take();
      }
    });
}
 

@Test
public void testCommit2() throws Exception {
  final Transaction transaction = channel.getTransaction();

  transaction.begin();
  transaction.rollback();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.commit();
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.commit();
      }
    });
}
 

@Test
public void testPut2() throws Exception {
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(events.get(0));
  transaction.rollback();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.put(events.get(0));
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.put(events.get(0));
      }
    });
}
 

@Test
public void shouldIndexOneEvent() throws Exception {
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  Transaction tx = channel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody("event #1 or 1".getBytes());
  channel.put(event);
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();
  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

  assertMatchAllQuery(1, event);
  assertBodyQuery(1, event);
}
 

@Test
public void testSingleEvent() throws Exception {
  StringEntity input = new StringEntity("[{\"headers\" : {\"a\": \"b\"},\"body\":"
          + " \"random_body\"}]");
  input.setContentType("application/json");
  postRequest.setEntity(input);

  httpClient.execute(postRequest);
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = channel.take();
  Assert.assertNotNull(e);
  Assert.assertEquals("b", e.getHeaders().get("a"));
  Assert.assertEquals("random_body", new String(e.getBody(),"UTF-8"));
  tx.commit();
  tx.close();
}
 

private void testFieldType(final String field, final String value, final Status result) {
  //System.out.println();
  headers.put(field, value);
  addEventToChannel(headers);
  boolean thrown = false;
  try {
    Status status = sink.process();
    Assert.assertEquals(result, status);
  } catch (EventDeliveryException ex) {
    thrown = true;
  }
  final Transaction tx = channel.getTransaction();
  tx.begin();
  final Event nextEvent = channel.take();
  tx.commit();
  tx.close();
  if (result == Status.READY) {
    Assert.assertFalse(thrown);
    Assert.assertNull(nextEvent);
  } else {
    Assert.assertTrue(thrown);
    Assert.assertNotNull(nextEvent);
  }
}
 

@Ignore @Test
public void processOneEvent() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  assertThat(sink.process()).isEqualTo(Sink.Status.READY);
  verify(table).save(ImmutableList.of(event));
  verifyNoMoreInteractions(table);
  verify(tx).begin();
  verify(tx).commit();
  verify(tx).close();
}
 

@Test
public void testClose2() throws Exception {
  final Transaction transaction = channel.getTransaction();

  testRuntimeException(new Runnable() {
      @Override
      public void run() {
        transaction.close();
      }
    });

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.close();
      }
    });
}
 

@Test
public void testClose3() throws Exception {
  final Transaction transaction = channel.getTransaction();

  testChannelException(new Runnable() {
      @Override
      public void run() {
        transaction.close();
      }
    });

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.close();
      }
    });
}
 

@Test (expected = EventDeliveryException.class)
public void testTimeOut() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration(),
    true);
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  Assert.fail();
}
 

@Test
public void testClose5() throws Exception {
  final Transaction transaction = channel.getTransaction();
  transaction.begin();

  testChannelException(new Runnable() {
      @Override
      public void run() {
        transaction.commit();
      }
    });

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.close();
      }
    });
}
 

@Test
public void testAppend() throws Exception {
  client = RpcClientFactory.getThriftInstance(props);
  Context context = new Context();
  channel.configure(context);
  configureSource();
  context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
  context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
  Configurables.configure(source, context);
  source.start();
  for(int i = 0; i < 30; i++) {
    client.append(EventBuilder.withBody(String.valueOf(i).getBytes()));
  }
  Transaction transaction = channel.getTransaction();
  transaction.begin();

  for (int i = 0; i < 30; i++) {
    Event event = channel.take();
    Assert.assertNotNull(event);
    Assert.assertEquals(String.valueOf(i), new String(event.getBody()));
  }
  transaction.commit();
  transaction.close();
}
 

@Test
public void testCommitAfterNoPutTake() throws Exception {
  channel.start();
  Assert.assertTrue(channel.isOpen());
  Transaction transaction;
  transaction = channel.getTransaction();
  transaction.begin();
  transaction.commit();
  transaction.close();

  // ensure we can reopen log with no error
  channel.stop();
  channel = createFileChannel();
  channel.start();
  Assert.assertTrue(channel.isOpen());
  transaction = channel.getTransaction();
  transaction.begin();
  Assert.assertNull(channel.take());
  transaction.commit();
  transaction.close();
}
 

@Test(expected=ChannelException.class)
public void testTransactionPutCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "2");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  // shouldn't be able to fit a third in the buffer
  channel.put(EventBuilder.withBody("test".getBytes()));
  Assert.fail();
}
 

@Test(expected=ChannelException.class)
public void testCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "3");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  // this should kill  it
  transaction.commit();
  Assert.fail();
}
 

@Override
public void processEvents(List<Event> events) {
	int batchSize = 10;
	int batches = events.size() / batchSize;
	for (int i = 0; i <= batches; i++) {
		Transaction txn = channel.getTransaction();
		txn.begin();
		int from = batchSize * i;
		int to = (i == batches) ? events.size() : from + batchSize;
		for (Event event : events.subList(from, to)) {
			channel.put(event);
			LOG.debug("Putting event to channel: {}", event);
		}
		txn.commit();
		txn.close();
		try {
			sink.process();
		} catch (EventDeliveryException e) {
			LOG.error("Error processing events!", e);
			throw new RuntimeException("Error processing events!", e);
		}
	}
}
 

@Test
public void testHeaders() throws InterruptedException, IOException {
  bind();

  Map flumeHeaders = new HashMap<CharSequence, ByteBuffer>();
  flumeHeaders.put("hello", ByteBuffer.wrap("world".getBytes("UTF-8")));
  ThriftFlumeEvent thriftEvent =  new ThriftFlumeEvent(
      1, Priority.INFO, ByteBuffer.wrap("foo".getBytes()),
      0, "fooHost", flumeHeaders);
  FlumeClient fClient = new FlumeClient("0.0.0.0", selectedPort);
  fClient.append(thriftEvent);

  // check if the event has arrived in the channel through OG thrift source
  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Event in channel has our header", "world",
      event.getHeaders().get("hello"));
  transaction.commit();
  transaction.close();

  stop();
}
 

@Test
public void TestOpenAndWriteSink() throws Exception {
    Map<String, Object> conf = Maps.newHashMap();
    StringSink stringSink = new StringSink();
    conf.put("name", "a1");
    conf.put("confFile", "./src/test/resources/flume/source.conf");
    conf.put("noReloadConf", false);
    conf.put("zkConnString", "");
    conf.put("zkBasePath", "");
    stringSink.open(conf, mockSinkContext);
    send(stringSink, 100);

    Thread.sleep(3 * 1000);
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    Event event = channel.take();

    Assert.assertNotNull(event);
    Assert.assertNotNull(mockRecord);

    verify(mockRecord, times(100)).ack();
    transaction.commit();
    transaction.close();
}
 

@Ignore @Test
public void processIllegalArgumentException() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  doThrow(IllegalArgumentException.class).when(table).save(anyListOf(Event.class));
  boolean hasThrown = false;
  try {
    sink.process();
  } catch (EventDeliveryException ex) {
    hasThrown = true;
    if (!(ex.getCause() instanceof IllegalArgumentException)) {
      fail("Did not throw inner IllegalArgumentException: " + ex);
    }
  }
  verify(tx).begin();
  verify(tx).rollback();
  verify(tx).close();
  verifyNoMoreInteractions(tx);
  if (!hasThrown) {
    fail("Did not throw exception");
  }
}
 

@Test
public void testClose4() throws Exception {
  final Transaction transaction = channel.getTransaction();
  transaction.begin();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.close();
      }
    });
}
 

@Test
public void testRollback5() throws Exception {
  final Transaction transaction = channel.getTransaction();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  transaction.begin();

  testInterrupt(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });
}
 

@Test
public void testCommit() throws InterruptedException, EventDeliveryException {

  Event event, event2;
  Context context = new Context();
  int putCounter = 0;

  context.put("keep-alive", "1");
  context.put("capacity", "100");
  context.put("transactionCapacity", "50");
  Configurables.configure(channel, context);

  Transaction transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  for (putCounter = 0; putCounter < 10; putCounter++) {
    event = EventBuilder.withBody(("test event" + putCounter).getBytes());
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 10; i++) {
    event2 = channel.take();
    Assert.assertNotNull("lost an event", event2);
    Assert.assertArrayEquals(event2.getBody(), ("test event" + i).getBytes());
    // System.out.println(event2.toString());
  }
  event2 = channel.take();
  Assert.assertNull("extra event found", event2);

  transaction.commit();
  transaction.close();
}
 

@Test
public void testRollback4() throws Exception {
  final Transaction transaction = channel.getTransaction();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  transaction.begin();

  testChannelException(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });
}
 

@Test
public void testSimple() throws IOException, InterruptedException {

  StringEntity input = new StringEntity("[{\"headers\":{\"a\": \"b\"},\"body\": \"random_body\"},"
          + "{\"headers\":{\"e\": \"f\"},\"body\": \"random_body2\"}]");
  //if we do not set the content type to JSON, the client will use
  //ISO-8859-1 as the charset. JSON standard does not support this.
  input.setContentType("application/json");
  postRequest.setEntity(input);

  HttpResponse response = httpClient.execute(postRequest);

  Assert.assertEquals(HttpServletResponse.SC_OK,
          response.getStatusLine().getStatusCode());
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = channel.take();
  Assert.assertNotNull(e);
  Assert.assertEquals("b", e.getHeaders().get("a"));
  Assert.assertEquals("random_body", new String(e.getBody(), "UTF-8"));

  e = channel.take();
  Assert.assertNotNull(e);
  Assert.assertEquals("f", e.getHeaders().get("e"));
  Assert.assertEquals("random_body2", new String(e.getBody(), "UTF-8"));
  tx.commit();
  tx.close();
}
 

@Override
public Status process() throws EventDeliveryException {
    List<Event> events;
    List<Map<String, Object>> parsedEvents;
    Status status = Status.BACKOFF;
    Transaction transaction = this.getChannel().getTransaction();
    try {
        transaction.begin();
        events = takeEventsFromChannel(this.getChannel(), batchSize);
        status = Status.READY;
        if (!events.isEmpty()) {
            updateSinkCounters(events);
            parsedEvents = eventParser.parse(events);
            sendEvents(parsedEvents);
            sinkCounter.addToEventDrainSuccessCount(events.size());
        } else {
            sinkCounter.incrementBatchEmptyCount();
        }
        transaction.commit();
        status = Status.READY;
    } catch (ChannelException e) {
        e.printStackTrace();
        transaction.rollback();
        status = Status.BACKOFF;
        this.sinkCounter.incrementConnectionFailedCount();
    } catch (Throwable t) {
        t.printStackTrace();
        transaction.rollback();
        status = Status.BACKOFF;
        if (t instanceof Error) {
            LOG.error(t.getMessage());
            throw new DruidSinkException("An error occurred during processing events to be stored in druid", t);

        }
    } finally {
        transaction.close();
    }
    return status;
}
 

@Test
public void testBegin() throws Exception {
  final Transaction transaction = channel.getTransaction();

  testExceptions(new Runnable() {
      @Override
      public void run() {
        transaction.begin();
      }
    });

  transaction.begin();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.begin();
      }
    });

  transaction.commit();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.begin();
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.begin();
      }
    });
}
 

@Test(expected = FlumeException.class)
public void testMissingTable() throws Exception {
  ctx.put("batchSize", "2");
  HBaseSink sink = new HBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 2);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 2; i++){
    for(int j = 0; j < 2; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(2, found);
  out = results[2];
  Assert.assertArrayEquals(Longs.toByteArray(2), out);
  sink.process();
  sink.stop();
}
 

@Test
public void shouldIndexFiveEvents() throws Exception {
  // Make it so we only need to call process once
  parameters.put(BATCH_SIZE, "5");
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  int numberOfEvents = 5;
  Event[] events = new Event[numberOfEvents];

  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < numberOfEvents; i++) {
    String body = "event #" + i + " of " + numberOfEvents;
    Event event = EventBuilder.withBody(body.getBytes());
    events[i] = event;
    channel.put(event);
  }
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();
  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

  assertMatchAllQuery(numberOfEvents, events);
  assertBodyQuery(5, events);
}
 

public static Transaction sharedInstance() {
  if (sharedInstance == null) {
    sharedInstance = new NoOpTransaction();
  }

  return sharedInstance;
}