Java Code Examples for org.apache.flume.Transaction#close()

private void takeWithEncoding(String encoding, int n, List<JSONEvent> events)
        throws Exception{
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = null;
  int i = 0;
  while (true) {
    e = channel.take();
    if (e == null) {
      break;
    }
    Event current = events.get(i++);
    Assert.assertEquals(new String(current.getBody(), encoding),
            new String(e.getBody(), encoding));
    Assert.assertEquals(current.getHeaders(), e.getHeaders());
  }
  Assert.assertEquals(n, events.size());
  tx.commit();
  tx.close();
}
 

private void handleTransactionFailure(Transaction txn)
    throws EventDeliveryException {
  try {
    txn.rollback();
  } catch (Throwable e) {
    logger.error("Failed to commit transaction." +
        "Transaction rolled back.", e);
    if(e instanceof Error || e instanceof RuntimeException){
      logger.error("Failed to commit transaction." +
          "Transaction rolled back.", e);
      Throwables.propagate(e);
    } else {
      logger.error("Failed to commit transaction." +
          "Transaction rolled back.", e);
      throw new EventDeliveryException("Failed to commit transaction." +
          "Transaction rolled back.", e);
    }
  } finally {
    txn.close();
  }
}
 

@Test (expected = EventDeliveryException.class)
public void testTimeOut() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration(),
    true);
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  Assert.fail();
}
 

@Test
public void testPutTake() throws InterruptedException, EventDeliveryException {
  Event event = EventBuilder.withBody("test event".getBytes());
  Context context = new Context();

  Configurables.configure(channel, context);

  Transaction transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  channel.put(event);
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  Event event2 = channel.take();
  Assert.assertEquals(event, event2);
  transaction.commit();
}
 

@Test
public void testRollbackAfterNoPutTake() throws Exception {
  channel.start();
  Assert.assertTrue(channel.isOpen());
  Transaction transaction;
  transaction = channel.getTransaction();
  transaction.begin();
  transaction.rollback();
  transaction.close();

  // ensure we can reopen log with no error
  channel.stop();
  channel = createFileChannel();
  channel.start();
  Assert.assertTrue(channel.isOpen());
  transaction = channel.getTransaction();
  transaction.begin();
  Assert.assertNull(channel.take());
  transaction.commit();
  transaction.close();
}
 

@Test
public void testWriteBinaryRecords() throws StageException {

  DataGeneratorFormatConfig dataGeneratorFormatConfig = new DataGeneratorFormatConfig();
  dataGeneratorFormatConfig.binaryFieldPath = "/data";
  FlumeTarget flumeTarget = FlumeTestUtil.createFlumeTarget(
      FlumeTestUtil.createDefaultFlumeConfig(port, false),
      DataFormat.BINARY,
      dataGeneratorFormatConfig
  );
  TargetRunner targetRunner = new TargetRunner.Builder(FlumeDTarget.class, flumeTarget).build();

  targetRunner.runInit();
  List<Record> logRecords = FlumeTestUtil.createBinaryRecords();
  targetRunner.runWrite(logRecords);
  targetRunner.runDestroy();

  for(Record r : logRecords) {
    Transaction transaction = ch.getTransaction();
    transaction.begin();
    Event event = ch.take();
    Assert.assertNotNull(event);
    Assert.assertTrue(Arrays.equals(r.get("/data").getValueAsByteArray(), event.getBody()));
    Assert.assertTrue(event.getHeaders().containsKey("charset"));
    Assert.assertEquals("UTF-8", event.getHeaders().get("charset"));
    transaction.commit();
    transaction.close();
  }
}
 

@Test
public void testCommit() throws InterruptedException, EventDeliveryException {

  Event event, event2;
  Context context = new Context();
  int putCounter = 0;

  context.put("keep-alive", "1");
  context.put("capacity", "100");
  context.put("transactionCapacity", "50");
  Configurables.configure(channel, context);

  Transaction transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  for (putCounter = 0; putCounter < 10; putCounter++) {
    event = EventBuilder.withBody(("test event" + putCounter).getBytes());
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 10; i++) {
    event2 = channel.take();
    Assert.assertNotNull("lost an event", event2);
    Assert.assertArrayEquals(event2.getBody(), ("test event" + i).getBytes());
    // System.out.println(event2.toString());
  }
  event2 = channel.take();
  Assert.assertNull("extra event found", event2);

  transaction.commit();
  transaction.close();
}
 

@Test
public void testMissingColumnsInEvent() throws EventDeliveryException, SQLException {
    
    final String fullTableName = "FLUME_TEST";
    initSinkContextWithDefaults(fullTableName);
  
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
  
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    
    sink.start();
    final String eventBody = "val1";
    final Event event = EventBuilder.withBody(Bytes.toBytes(eventBody));
    // put event in channel
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    channel.put(event);
    transaction.commit();
    transaction.close();

    sink.process();
    
    int rowsInDb = countRows(fullTableName);
    assertEquals(0 , rowsInDb);
       
    sink.stop();
    assertEquals(LifecycleState.STOP, sink.getLifecycleState());
    
}
 

@Test
public void testThreeEvents() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 3; i++){
    for(int j = 0; j < 3; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
}
 

@Test
public void shouldAllowCustomElasticSearchIndexRequestBuilderFactory()
    throws Exception {

  parameters.put(SERIALIZER,
      CustomElasticSearchIndexRequestBuilderFactory.class.getName());

  Configurables.configure(fixture, new Context(parameters));

  Channel channel = bindAndStartChannel(fixture);
  Transaction tx = channel.getTransaction();
  tx.begin();
  String body = "{ foo: \"bar\" }";
  Event event = EventBuilder.withBody(body.getBytes());
  channel.put(event);
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();

  assertEquals(fixture.getIndexName()+"-05_17_36_789",
      CustomElasticSearchIndexRequestBuilderFactory.actualIndexName);
  assertEquals(fixture.getIndexType(),
      CustomElasticSearchIndexRequestBuilderFactory.actualIndexType);
  assertArrayEquals(event.getBody(),
      CustomElasticSearchIndexRequestBuilderFactory.actualEventBody);
  assertTrue(CustomElasticSearchIndexRequestBuilderFactory.hasContext);
}
 

@Test
public void testWriteStringRecords() throws StageException {

  DataGeneratorFormatConfig dataGeneratorFormatConfig = new DataGeneratorFormatConfig();
  dataGeneratorFormatConfig.textFieldPath = "/";
  dataGeneratorFormatConfig.charset = "UTF-8";
  dataGeneratorFormatConfig.textEmptyLineIfNull = true;

  FlumeTarget flumeTarget = FlumeTestUtil.createFlumeTarget(
    FlumeTestUtil.createDefaultFlumeConfig(port, false),
    DataFormat.TEXT,
    dataGeneratorFormatConfig
  );
  TargetRunner targetRunner = new TargetRunner.Builder(FlumeDTarget.class, flumeTarget).build();

  targetRunner.runInit();
  List<Record> logRecords = FlumeTestUtil.createStringRecords();
  targetRunner.runWrite(logRecords);
  targetRunner.runDestroy();

  for(Record r : logRecords) {
    Transaction transaction = ch.getTransaction();
    transaction.begin();
    Event event = ch.take();
    Assert.assertNotNull(event);
    Assert.assertEquals(r.get().getValueAsString(), new String(event.getBody()).trim());
    Assert.assertTrue(event.getHeaders().containsKey("charset"));
    Assert.assertEquals("UTF-8", event.getHeaders().get("charset"));
    transaction.commit();
    transaction.close();
  }
}
 

@Test
public void testWriteStringRecordsFromJSON() throws InterruptedException, StageException, IOException {

  DataGeneratorFormatConfig dataGeneratorFormatConfig = new DataGeneratorFormatConfig();
  dataGeneratorFormatConfig.textFieldPath = "/name";
  dataGeneratorFormatConfig.charset = "UTF-8";
  dataGeneratorFormatConfig.textEmptyLineIfNull = true;

  FlumeTarget flumeTarget = FlumeTestUtil.createFlumeTarget(
    FlumeTestUtil.createDefaultFlumeConfig(port, false),
    DataFormat.TEXT,
    dataGeneratorFormatConfig
  );

  TargetRunner targetRunner = new TargetRunner.Builder(FlumeDTarget.class, flumeTarget).build();

  targetRunner.runInit();
  List<Record> logRecords = FlumeTestUtil.createJsonRecords();
  targetRunner.runWrite(logRecords);
  targetRunner.runDestroy();

  for(Record r : logRecords) {
    Transaction transaction = ch.getTransaction();
    transaction.begin();
    Event event = ch.take();
    Assert.assertNotNull(event);
    Assert.assertEquals(r.get().getValueAsMap().get("name").getValueAsString(), new String(event.getBody()).trim());
    Assert.assertTrue(event.getHeaders().containsKey("charset"));
    Assert.assertEquals("UTF-8", event.getHeaders().get("charset"));
    transaction.commit();
    transaction.close();
  }
}
 

private void addEventToChannel(Map<String, String> headers) {
  Event event = EventBuilder.withBody("body", Charsets.UTF_8, headers);
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(event);
  transaction.commit();
  transaction.close();
}
 

@Test
public void testCapacityBufferEmptyingAfterTakeCommit() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "3");
  parms.put("transactionCapacity", "3");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  tx.commit();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.take();
  channel.take();
  tx.commit();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  tx.commit();
  tx.close();
}
 

@Test
public void testByteCapacityBufferEmptyingAfterRollback() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("byteCapacity", "2000");
  parms.put("byteCapacityBufferPercentage", "20");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  byte[] eventBody = new byte[405];

  Transaction tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  tx.rollback();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  tx.commit();
  tx.close();
}
 

@Test
public void testRollbackSimulatedCrashWithSink() throws Exception {
  channel.start();
  Assert.assertTrue(channel.isOpen());
  int numEvents = 100;

  LoggerSink sink = new LoggerSink();
  sink.setChannel(channel);
  // sink will leave one item
  CountingSinkRunner runner = new CountingSinkRunner(sink, numEvents - 1);
  runner.start();
  putEvents(channel, "rollback", 10, numEvents);

  Transaction transaction;
  // put an item we will rollback
  transaction = channel.getTransaction();
  transaction.begin();
  byte[] bytes = "rolled back".getBytes(Charsets.UTF_8);
  channel.put(EventBuilder.withBody(bytes));
  transaction.rollback();
  transaction.close();

  while(runner.isAlive()) {
    Thread.sleep(10L);
  }
  Assert.assertEquals(numEvents - 1, runner.getCount());
  for(Exception ex : runner.getErrors()) {
    LOG.warn("Sink had error", ex);
  }
  Assert.assertEquals(Collections.EMPTY_LIST, runner.getErrors());

  // simulate crash
  channel.stop();
  channel = createFileChannel();
  channel.start();
  Assert.assertTrue(channel.isOpen());
  Set<String> out = takeEvents(channel, 1, 1);
  Assert.assertEquals(1, out.size());
  String s = out.iterator().next();
  Assert.assertTrue(s, s.startsWith("rollback-90-9"));
}
 

@Test
public void testMultipleClients() throws Exception {
  ExecutorService submitter = Executors.newCachedThreadPool();
  client = RpcClientFactory.getThriftInstance(props);
  Context context = new Context();
  context.put("capacity", "1000");
  context.put("transactionCapacity", "1000");
  channel.configure(context);
  configureSource();
  context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
  context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
  Configurables.configure(source, context);
  source.start();
  ExecutorCompletionService<Void> completionService = new
    ExecutorCompletionService(submitter);
  for (int i = 0; i < 30; i++) {
    completionService.submit(new SubmitHelper(i), null);
  }
  //wait for all threads to be done


  for(int i = 0; i < 30; i++) {
    completionService.take();
  }

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  long after = System.currentTimeMillis();
  List<Integer> events = Lists.newArrayList();
  for (int i = 0; i < 300; i++) {
    Event event = channel.take();
    Assert.assertNotNull(event);
    Assert.assertTrue(Long.valueOf(event.getHeaders().get("time")) < after);
    events.add(Integer.parseInt(new String(event.getBody())));
  }
  transaction.commit();
  transaction.close();

  Collections.sort(events);

  int index = 0;
  //30 batches of 10
  for(int i = 0; i < 30; i++) {
    for(int j = 0; j < 10; j++) {
      Assert.assertEquals(i, events.get(index++).intValue());
    }
  }
}
 

@Test
public void testAppend() throws InterruptedException, LifecycleException,
    EventDeliveryException, IOException {

  LOG.debug("Starting...");
  final long rollCount = 3;
  final long batchSize = 2;
  final String fileName = "FlumeData";

  // clear the test directory
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.get(conf);
  Path dirPath = new Path(testPath);
  fs.delete(dirPath, true);
  fs.mkdirs(dirPath);

  Context context = new Context();

  context.put("hdfs.path", testPath + "/%Y-%m-%d/%H");
  context.put("hdfs.timeZone", "UTC");
  context.put("hdfs.filePrefix", fileName);
  context.put("hdfs.rollCount", String.valueOf(rollCount));
  context.put("hdfs.batchSize", String.valueOf(batchSize));

  Configurables.configure(sink, context);

  Channel channel = new MemoryChannel();
  Configurables.configure(channel, context);

  sink.setChannel(channel);
  sink.start();

  Calendar eventDate = Calendar.getInstance();
  List<String> bodies = Lists.newArrayList();
  // push the event batches into channel
  for (int i = 1; i < 4; i++) {
    Transaction txn = channel.getTransaction();
    txn.begin();
    for (int j = 1; j <= batchSize; j++) {
      Event event = new SimpleEvent();
      eventDate.clear();
      eventDate.set(2011, i, i, i, 0); // yy mm dd
      event.getHeaders().put("timestamp",
          String.valueOf(eventDate.getTimeInMillis()));
      event.getHeaders().put("hostname", "Host" + i);
      String body = "Test." + i + "." + j;
      event.setBody(body.getBytes());
      bodies.add(body);
      channel.put(event);
    }
    txn.commit();
    txn.close();

    // execute sink to process the events
    sink.process();
  }

  sink.stop();
  verifyOutputSequenceFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
}
 

@Test
    public void templateWithH2() throws Exception {
        Class.forName("org.h2.Driver").newInstance();
        Connection conn = DriverManager.getConnection("jdbc:h2:/tmp/jdbcsink_test");
        conn.prepareStatement("DROP TABLE public.test IF EXISTS;").execute();
        conn.prepareStatement("CREATE TABLE public.test (myInteger INTEGER, myString VARCHAR, myId BIGINT AUTO_INCREMENT PRIMARY KEY);").execute();
        conn.commit();
        conn.close();

        Context ctx = new Context();
        ctx.put("driver", "org.h2.Driver");
        ctx.put("connectionString", "jdbc:h2:/tmp/jdbcsink_test");
        ctx.put("sqlDialect", "H2");
        ctx.put("batchSize", "1");
        ctx.put("sql", "INSERT INTO \"PUBLIC\".\"TEST\" (\"MYINTEGER\", \"MYSTRING\") VALUES (cast(${header.myInteger:int} as integer), cast(${header.myString:varchar} as varchar))");

        JDBCSink jdbcSink = new JDBCSink();

        Configurables.configure(jdbcSink, ctx);

        Context channelContext = new Context();
        channelContext.put("capacity", "10000");
        channelContext.put("transactionCapacity", "200");

        Channel channel = new MemoryChannel();
        channel.setName("junitChannel");
        Configurables.configure(channel, channelContext);

        jdbcSink.setChannel(channel);

        channel.start();
        jdbcSink.start();

        Transaction tx = channel.getTransaction();
        tx.begin();

        Map<String, String> headers = new HashMap<String, String>();
        headers.put("myString", "bar"); // Overwrites the value defined in JSON body
        headers.put("myInteger", "64");
        headers.put("myBoolean", "true");
        headers.put("myDouble", "1.0");
        headers.put("myNull", "foobar");

        Date myDate = new Date();
        headers.put("myDate", Long.toString(myDate.getTime()));

        headers.put("myString2", "baz");

        Event event = EventBuilder.withBody(new byte[0], headers);
        channel.put(event);

        tx.commit();
        tx.close();

        jdbcSink.process();

        jdbcSink.stop();
        channel.stop();

        conn = DriverManager.getConnection("jdbc:h2:/tmp/jdbcsink_test");

        ResultSet rs = conn.prepareStatement("SELECT count(*) AS count FROM public.test").executeQuery();
        rs.next();
        assertThat(rs.getInt("count")).isEqualTo(1);

        rs = conn.prepareStatement("SELECT * FROM public.test").executeQuery();
        rs.next();
//        for (int i = 1; i <= 3; i++) {
//            System.out.println(rs.getString(i));
//        }
        assertThat(rs.getInt("myInteger")).isEqualTo(64);
        assertThat(rs.getString("myString")).isEqualTo("bar");
        conn.close();

    }
 

@Test
public void testSslProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}