Java Code Examples for org.apache.flume.Channel#getTransaction()

@Ignore @Test
public void shouldIndexOneEvent() throws Exception {
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  Transaction tx = channel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody("event #1 or 1".getBytes());
  channel.put(event);
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();
  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

  assertMatchAllQuery(1, event);
  assertBodyQuery(1, event);
}
 

@Test (expected = EventDeliveryException.class)
public void testTimeOut() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration(),
    true);
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  Assert.fail();
}
 

@Override
public Status process() throws EventDeliveryException {
  Status result = Status.READY;
  Channel channel = getChannel();
  Transaction transaction = channel.getTransaction();
  Event event = null;

  try {
    transaction.begin();
    event = channel.take();

    if (event != null) {
      if (logger.isInfoEnabled()) {
        logger.info("Event: " + EventHelper.dumpEvent(event));
      }
    } else {
      // No event found, request back-off semantics from the sink runner
      result = Status.BACKOFF;
    }
    transaction.commit();
  } catch (Exception ex) {
    transaction.rollback();
    throw new EventDeliveryException("Failed to log event: " + event, ex);
  } finally {
    transaction.close();
  }

  return result;
}
 

@Test(expected = FlumeException.class)
public void testMissingTable() throws Exception {
  deleteTable = false;
  ctx.put("batchSize", "2");
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  Assert.assertFalse(sink.isConfNull());
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 2);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 2; i++){
    for(int j = 0; j < 2; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(2, found);
  out = results[2];
  Assert.assertArrayEquals(Longs.toByteArray(2), out);
  sink.process();
  sink.stop();
}
 

@Test
public void testOneEvent() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = EventBuilder.withBody(
      Bytes.toBytes(valBase));
  channel.put(e);
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 1);
  byte[] out = results[0];
  Assert.assertArrayEquals(e.getBody(), out);
  out = results[1];
  Assert.assertArrayEquals(Longs.toByteArray(1), out);
}
 

@Test
public void testThreeEvents() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 3; i++){
    for(int j = 0; j < 3; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
}
 

@Override
public Status process() throws EventDeliveryException {
    Channel channel = getChannel();
    Transaction tx = channel.getTransaction();
    try {
        tx.begin();
        Event event = channel.take();
        if (event == null) {
            tx.commit();
            return Status.READY;

        }

        String data = null;
        if(writeBody){
            data = new String(event.getBody());
        } else {
            data = mapper.writeValueAsString(event.getHeaders());
        }

        producer.send(new KeyedMessage<String, String>(topic, data));
        tx.commit();
        return Status.READY;
    } catch (Exception e) {
        try {
            tx.rollback();
            return Status.BACKOFF;
        } catch (Exception e2) {
            log.error("Rollback Exception:{}", e2);
        }
        log.error("KafkaSink Exception:{}", e);
        return Status.BACKOFF;
    } finally {
        tx.close();
    }
}
 

@Test
public void shouldIndexFiveEventsOverThreeBatches() throws Exception {
  parameters.put(BATCH_SIZE, "2");
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  int numberOfEvents = 5;
  Event[] events = new Event[numberOfEvents];

  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < numberOfEvents; i++) {
    String body = "event #" + i + " of " + numberOfEvents;
    Event event = EventBuilder.withBody(body.getBytes());
    events[i] = event;
    channel.put(event);
  }
  tx.commit();
  tx.close();

  int count = 0;
  Status status = Status.READY;
  while (status != Status.BACKOFF) {
    count++;
    status = fixture.process();
  }
  fixture.stop();

  assertEquals(3, count);

  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();
  assertMatchAllQuery(numberOfEvents, events);
  assertBodyQuery(5, events);
}
 

@Override
public Status process() throws EventDeliveryException {
  Status status = Status.READY;

  Channel channel = getChannel();
  Transaction transaction = channel.getTransaction();
  Event event = null;
  long eventCounter = counterGroup.get("events.success");

  try {
    transaction.begin();
    int i = 0;
    for (i = 0; i < batchSize; i++) {
      event = channel.take();
      if (++eventCounter % logEveryNEvents == 0) {
        logger.info("Null sink {} successful processed {} events.", getName(), eventCounter);
      }
      if(event == null) {
        status = Status.BACKOFF;
        break;
      }
    }
    transaction.commit();
    counterGroup.addAndGet("events.success", (long) Math.min(batchSize, i));
    counterGroup.incrementAndGet("transaction.success");
  } catch (Exception ex) {
    transaction.rollback();
    counterGroup.incrementAndGet("transaction.failed");
    logger.error("Failed to deliver event. Exception follows.", ex);
    throw new EventDeliveryException("Failed to deliver event: " + event, ex);
  } finally {
    transaction.close();
  }

  return status;
}
 

@Test
public void testMissingColumnsInEvent() throws EventDeliveryException, SQLException {
    
    final String fullTableName = "FLUME_TEST";
    initSinkContextWithDefaults(fullTableName);
  
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
  
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    
    sink.start();
    final String eventBody = "val1";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();

    sink.process();
    
    int rowsInDb = countRows(fullTableName);
    assertEquals(0 , rowsInDb);
       
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    
}
 

@Test
public void shouldAllowCustomElasticSearchIndexRequestBuilderFactory()
    throws Exception {

  parameters.put(SERIALIZER,
      CustomElasticSearchIndexRequestBuilderFactory.class.getName());

  Configurables.configure(fixture, new Context(parameters));

  Channel channel = bindAndStartChannel(fixture);
  Transaction tx = channel.getTransaction();
  tx.begin();
  String body = "{ foo: \"bar\" }";
  Event event = EventBuilder.withBody(body.getBytes());
  channel.put(event);
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();

  assertEquals(fixture.getIndexName()+"-05_17_36_789",
      CustomElasticSearchIndexRequestBuilderFactory.actualIndexName);
  assertEquals(fixture.getIndexType(),
      CustomElasticSearchIndexRequestBuilderFactory.actualIndexType);
  assertArrayEquals(event.getBody(),
      CustomElasticSearchIndexRequestBuilderFactory.actualEventBody);
  assertTrue(CustomElasticSearchIndexRequestBuilderFactory.hasContext);
}
 

@Test
public void testKeyGenerator() throws EventDeliveryException, SQLException {
    
    final String fullTableName = "FLUME_TEST";
    initSinkContextWithDefaults(fullTableName);
    
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
 
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
  
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    
    sink.start();
    final String eventBody = "val1\tval2";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();

    sink.process();
   
    int rowsInDb = countRows(fullTableName);
    assertEquals(1 , rowsInDb);
    
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
     
}
 

@Override
public Status process() throws EventDeliveryException {
  Status status = Status.READY;
  Channel channel = getChannel();
  Transaction transaction = channel.getTransaction();

  resetLock.lock();
  try {
    transaction.begin();

    verifyConnection();

    List<Event> batch = Lists.newLinkedList();

    for (int i = 0; i < client.getBatchSize(); i++) {
      Event event = channel.take();

      if (event == null) {
        break;
      }

      batch.add(event);
    }

    int size = batch.size();
    int batchSize = client.getBatchSize();
    logger.info("RpcSink " + getName() + " consume " + size + ", want " + batchSize);

    if (size == 0) {
      sinkCounter.incrementBatchEmptyCount();
      status = Status.BACKOFF;
    } else {
      if (size < batchSize) {
        sinkCounter.incrementBatchUnderflowCount();
      } else {
        sinkCounter.incrementBatchCompleteCount();
      }
      sinkCounter.addToEventDrainAttemptCount(size);
      client.appendBatch(batch);
    }

    transaction.commit();
    sinkCounter.addToEventDrainSuccessCount(size);

  } catch (Throwable t) {
    transaction.rollback();
    if (t instanceof Error) {
      throw (Error) t;
     } else if (t instanceof ChannelException) {
      logger.error("Rpc Sink " + getName() + ": Unable to get event from" +
          " channel " + channel.getName() + ". Exception follows.", t);
      status = Status.BACKOFF;
    } else {
      destroyConnection();
      throw new EventDeliveryException("Failed to send events", t);
    }
  } finally {
    resetLock.unlock();
    transaction.close();
  }

  return status;
}
 

@Test
public void testApacheLogRegex() throws Exception {
    
    sinkContext = new Context ();
    final String fullTableName = "s1.apachelogs";
    final String logRegex = "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) \"([^ ]+) ([^ ]+)" +
                            " ([^\"]+)\" (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\")" +
                            " ([^ \"]*|\"[^\"]*\"))?";
    
    final String columns = "host,identity,user,time,method,request,protocol,status,size,referer,agent";
    
    String ddl = "CREATE TABLE " + fullTableName +
            "  (uid VARCHAR NOT NULL, user VARCHAR, time varchar, host varchar , identity varchar, method varchar, request varchar , protocol varchar," +
            "  status integer , size integer , referer varchar , agent varchar CONSTRAINT pk PRIMARY KEY (uid))\n";
   
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER,EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION,logRegex);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES,columns);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR,DefaultKeyGenerator.UUID.name());
   
    String message1 = "33.22.11.00 - user1 [12/Dec/2013:07:01:19 +0000] " +
            "\"GET /wp-admin/css/install.css HTTP/1.0\" 200 813 " + 
            "\"http://www.google.com\" \"Mozilla/5.0 (comp" +
            "atible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)\"";
    
    String message2 = "192.168.20.1 - user2 [13/Dec/2013:06:05:19 +0000] " +
            "\"GET /wp-admin/css/install.css HTTP/1.0\" 400 363 " + 
            "\"http://www.salesforce.com/in/?ir=1\" \"Mozilla/5.0 (comp" +
            "atible;)\"";
    
    
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
  
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    
    sink.start();
    
    final Event event1 = EventBuilder.withBody(Bytes.toBytes(message1));
    final Event event2 = EventBuilder.withBody(Bytes.toBytes(message2));
    
    final Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event1);
    channel.put(event2);
    transaction.commit();
    transaction.close();

    sink.process();
   
    final String query = " SELECT * FROM \n " + fullTableName;
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    final ResultSet rs ;
    final Connection conn = DriverManager.getConnection(getUrl(), props);
    try{
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertTrue(rs.next());
         
    }finally {
        if(conn != null) {
            conn.close();
        }
    }
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    
}
 

@Test
	public void testProcess() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
		Channel channel = new MemoryChannel();
		Context context = new Context();

		String resource = "src/test/resources/server.log";
		assertNotNull(resource);

		int expectedNrOfEvents = 0;

		BufferedReader br = new BufferedReader(new FileReader(resource));
		String line;
		while ((line = br.readLine()) != null) {
			if (line.endsWith("|#]")) {
				expectedNrOfEvents++;
			}
		}

		context.put("command", format("cat %s", resource));
		context.put("event.terminator", "|#]");
		context.put("keep-alive", "1");
		context.put("capacity", "100000");
		context.put("transactionCapacity", "100000");
		Configurables.configure(source, context);
		Configurables.configure(channel, context);

		ChannelSelector rcs = new ReplicatingChannelSelector();
		rcs.setChannels(Lists.newArrayList(channel));

		source.setChannelProcessor(new ChannelProcessor(rcs));

		source.start();
		Transaction transaction = channel.getTransaction();

		transaction.begin();
		Event event;

		int actualNrOfEvents = 0;

		FileOutputStream outputStream = new FileOutputStream("/tmp/flume-execsource." + Thread.currentThread().getId());

		while ((event = channel.take()) != null) {
			outputStream.write(event.getBody());
			outputStream.write('\n');
			outputStream.write('\n');
			actualNrOfEvents++;
		}

		outputStream.close();
		transaction.commit();
		transaction.close();

		source.stop();

		File actualFile = new File("/tmp/flume-execsource." + Thread.currentThread().getId());
		File expectedFile = new File(resource);

		assertEquals(expectedNrOfEvents, actualNrOfEvents);

        // This doesn't work anymore since we implemented the different event separator § instead of \n
//		assertEquals(FileUtils.checksumCRC32(expectedFile), FileUtils.checksumCRC32(actualFile));

		FileUtils.forceDelete(actualFile);
	}
 

@Override
public Status process() throws EventDeliveryException {
  logger.debug("processing...");
  Status status = Status.READY;
  Channel channel = getChannel();
  Transaction txn = channel.getTransaction();
  try {
    txn.begin();
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (int i = 0; i < batchSize; i++) {
      Event event = channel.take();

      if (event == null) {
        break;
      }

      IndexRequestBuilder indexRequest =
          indexRequestFactory.createIndexRequest(
              client, indexName, indexType, event);

      if (ttlMs > 0) {
        indexRequest.setTTL(ttlMs);
      }

      bulkRequest.add(indexRequest);
    }

    int size = bulkRequest.numberOfActions();
    if (size <= 0) {
      sinkCounter.incrementBatchEmptyCount();
      counterGroup.incrementAndGet("channel.underflow");
      status = Status.BACKOFF;
    } else {
      if (size < batchSize) {
        sinkCounter.incrementBatchUnderflowCount();
        status = Status.BACKOFF;
      } else {
        sinkCounter.incrementBatchCompleteCount();
      }

      sinkCounter.addToEventDrainAttemptCount(size);

      BulkResponse bulkResponse = bulkRequest.execute().actionGet();
      if (bulkResponse.hasFailures()) {
        throw new EventDeliveryException(bulkResponse.buildFailureMessage());
      }
    }
    txn.commit();
    sinkCounter.addToEventDrainSuccessCount(size);
    counterGroup.incrementAndGet("transaction.success");
  } catch (Throwable ex) {
    try {
      txn.rollback();
      counterGroup.incrementAndGet("transaction.rollback");
    } catch (Exception ex2) {
      logger.error(
          "Exception in rollback. Rollback might not have been successful.",
          ex2);
    }

    if (ex instanceof Error || ex instanceof RuntimeException) {
      logger.error("Failed to commit transaction. Transaction rolled back.",
          ex);
      Throwables.propagate(ex);
    } else {
      logger.error("Failed to commit transaction. Transaction rolled back.",
          ex);
      throw new EventDeliveryException(
          "Failed to commit transaction. Transaction rolled back.", ex);
    }
  } finally {
    txn.close();
  }
  return status;
}
 

@Test
public void testApacheLogRegex() throws Exception {
    
    sinkContext = new Context ();
    final String fullTableName = "s1.apachelogs";
    final String logRegex = "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) \"([^ ]+) ([^ ]+)" +
                            " ([^\"]+)\" (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\")" +
                            " ([^ \"]*|\"[^\"]*\"))?";
    
    final String columns = "host,identity,user,time,method,request,protocol,status,size,referer,agent";
    
    String ddl = "CREATE TABLE " + fullTableName +
            "  (uid VARCHAR NOT NULL, user VARCHAR, time varchar, host varchar , identity varchar, method varchar, request varchar , protocol varchar," +
            "  status integer , size integer , referer varchar , agent varchar CONSTRAINT pk PRIMARY KEY (uid))\n";
   
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, TestUtil.PHOENIX_JDBC_URL);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER,EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION,logRegex);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES,columns);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR,DefaultKeyGenerator.UUID.name());
   
    String message1 = "33.22.11.00 - user1 [12/Dec/2013:07:01:19 +0000] " +
            "\"GET /wp-admin/css/install.css HTTP/1.0\" 200 813 " + 
            "\"http://www.google.com\" \"Mozilla/5.0 (comp" +
            "atible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)\"";
    
    String message2 = "192.168.20.1 - user2 [13/Dec/2013:06:05:19 +0000] " +
            "\"GET /wp-admin/css/install.css HTTP/1.0\" 400 363 " + 
            "\"http://www.salesforce.com/in/?ir=1\" \"Mozilla/5.0 (comp" +
            "atible;)\"";
    
    
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
  
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    
    sink.start();
    
    final Event event1 = EventBuilder.withBody(Bytes.toBytes(message1));
    final Event event2 = EventBuilder.withBody(Bytes.toBytes(message2));
    
    final Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event1);
    channel.put(event2);
    transaction.commit();
    transaction.close();

    sink.process();
   
    final String query = " SELECT * FROM \n " + fullTableName;
    Properties props = new Properties(TEST_PROPERTIES);
    final ResultSet rs ;
    final Connection conn = DriverManager.getConnection(getUrl(), props);
    try{
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertTrue(rs.next());
         
    }finally {
        if(conn != null) {
            conn.close();
        }
    }
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    
}
 

@Test
public void testWriteStringRecordsRoundRobin() throws StageException {

  DataGeneratorFormatConfig dataGeneratorFormatConfig = new DataGeneratorFormatConfig();
  dataGeneratorFormatConfig.textFieldPath = "/";
  dataGeneratorFormatConfig.textEmptyLineIfNull = false;

  FlumeTarget flumeTarget = FlumeTestUtil.createFlumeTarget(
    FlumeTestUtil.createFlumeConfig(
      false,                      // backOff
      100,                          // batchSize
      ClientType.AVRO_LOAD_BALANCING,
      2000,                       // connection timeout
      flumeHosts,
      HostSelectionStrategy.ROUND_ROBIN,
      -1,                          // maxBackOff
      1,                          // maxRetryAttempts
      2000,                       // requestTimeout
      false,                      // singleEventPerBatch
      0
    ),
    DataFormat.TEXT,
    dataGeneratorFormatConfig
  );

  TargetRunner targetRunner = new TargetRunner.Builder(FlumeDTarget.class, flumeTarget).build();

  targetRunner.runInit();
  List<List<Record>> logRecords = new ArrayList<>(NUM_HOSTS);
  for(int i = 0; i < NUM_HOSTS; i++) {
    logRecords.add(FlumeTestUtil.createStringRecords());
  }

  for(int i = 0; i < NUM_HOSTS; i++) {
    targetRunner.runWrite(logRecords.get(i));
  }
  targetRunner.runDestroy();

  for(int i = 0;i < logRecords.size(); i++) {
    Channel channel = chs.get(i % NUM_HOSTS);
    List<Record> records = logRecords.get(i);
    for(int j = 0; j < records.size(); j++) {
      Transaction transaction = channel.getTransaction();
      transaction.begin();
      Event event = channel.take();
      Assert.assertNotNull(event);
      Assert.assertEquals(records.get(j).get().getValueAsString(), new String(event.getBody()).trim());
      Assert.assertTrue(event.getHeaders().containsKey("charset"));
      Assert.assertEquals("UTF-8", event.getHeaders().get("charset"));
      transaction.commit();
      transaction.close();
    }
  }
}
 

@Test
public void testMultipleBatches() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  ctx.put("batchSize", "2");
  HBaseSink sink = new HBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  int count = 0;
  Status status = Status.READY;
  while(status != Status.BACKOFF){
    count++;
    status = sink.process();
  }
  sink.stop();
  Assert.assertEquals(2, count);
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 3; i++){
    for(int j = 0; j < 3; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
  testUtility.deleteTable(tableName.getBytes());
}
 

/**
 * {@inheritDoc}
 */
@Override
public Status process() throws EventDeliveryException {
  log.debug("Executing CassandraSink.process()...");
  Status status = Status.READY;
  Channel channel = getChannel();
  Transaction txn = channel.getTransaction();
  try {
    txn.begin();
    int count;
    List<Event> eventList= new ArrayList<Event>();
    for (count = 0; count < batchSize; ++count) {
      Event event = channel.take();

      if (event == null) {
        break;
      }
      eventList.add(event);
    }

    if (count <= 0) {
      sinkCounter.incrementBatchEmptyCount();
      counterGroup.incrementAndGet("channel.underflow");
      status = Status.BACKOFF;
    } else {
      if (count < batchSize) {
        sinkCounter.incrementBatchUnderflowCount();
        status = Status.BACKOFF;
      } else {
        sinkCounter.incrementBatchCompleteCount();
      }

      for (final CassandraTable table : tables) {
        table.save(eventList);
      }

      sinkCounter.addToEventDrainAttemptCount(count);
      //client.execute();
    }
    txn.commit();
    sinkCounter.addToEventDrainSuccessCount(count);
    counterGroup.incrementAndGet("transaction.success");
  } catch (Throwable ex) {
    try {
      txn.rollback();
      counterGroup.incrementAndGet("transaction.rollback");
    } catch (Exception ex2) {
      log.error(
              "Exception in rollback. Rollback might not have been successful.",
              ex2);
    }

    if (ex instanceof Error || ex instanceof RuntimeException) {
      log.error("Failed to commit transaction. Transaction rolled back.",
              ex);
      Throwables.propagate(ex);
    } else {
      log.error("Failed to commit transaction. Transaction rolled back.",
              ex);
      throw new EventDeliveryException(
              "Failed to commit transaction. Transaction rolled back.", ex);
    }
  } finally {
    txn.close();
  }
  return status;

}