org.apache.flume.conf.Configurables Java Examples

@OnScheduled
public void onScheduled(final ProcessContext context) {
    try {
        channel = new NifiSinkSessionChannel(SUCCESS, FAILURE);
        channel.start();

        sink = SINK_FACTORY.create(context.getProperty(SOURCE_NAME).getValue(),
                context.getProperty(SINK_TYPE).getValue());
        sink.setChannel(channel);

        String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
        String agentName = context.getProperty(AGENT_NAME).getValue();
        String sinkName = context.getProperty(SOURCE_NAME).getValue();
        Configurables.configure(sink,
                getFlumeSinkContext(flumeConfig, agentName, sinkName));

        sink.start();
    } catch (Throwable th) {
        getLogger().error("Error creating sink", th);
        throw Throwables.propagate(th);
    }
}
 

@Before
public void initiate() throws Exception{
  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  File TESTFILE = new File(
      TestLog4jAppender.class.getClassLoader()
          .getResource("flume-log4jtest.properties").getFile());
  FileReader reader = new FileReader(TESTFILE);
  props = new Properties();
  props.load(reader);
  reader.close();
}
 

@Before
public void setup() {
    //        Context channelContext = new Context();
    //        channelContext.put("checkpointDir","data/check");
    //        channelContext.put("dataDirs","data/data");
    //        channelContext.put("capacity","1000");
    //        channelContext.put("transactionCapacity","100");
    //        channelContext.put("checkpointInterval","300");
    //        channel = new FileChannel();
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "5000");
    channel = new MemoryChannel();
    channel.setName("junitChannel");
    Configurables.configure(channel, channelContext);
    channel.start();

    druidSink = new DruidSink();
    druidSink.setChannel(channel);
    druidSink.configure(getMockContext());
    druidSink.start();
}
 

private Sink.Status prepareAndSend(Context context, String msg) throws EventDeliveryException {
    Sink kafkaSink = new KafkaSink();
    Configurables.configure(kafkaSink, context);
    Channel memoryChannel = new MemoryChannel();
    Configurables.configure(memoryChannel, context);
    kafkaSink.setChannel(memoryChannel);
    kafkaSink.start();

    Transaction tx = memoryChannel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody(msg.getBytes());
    memoryChannel.put(event);
    tx.commit();
    tx.close();

    return kafkaSink.process();
}
 

private void createSparkAvroSink() {
	sparkAvroChannel = new MemoryChannel();
	Map<String, String> channelParamters = new HashMap<>();
	channelParamters.put("capacity", "100000");
	channelParamters.put("transactionCapacity", "1000");
	Context channelContext = new Context(channelParamters);
	Configurables.configure(sparkAvroChannel, channelContext);
	String channelName = "SparkAvroMemoryChannel-" + UUID.randomUUID();
	sparkAvroChannel.setName(channelName);

	sparkAvroSink = new AvroSink();
	sparkAvroSink.setName("SparkAvroSink-" + UUID.randomUUID());
	Map<String, String> paramters = new HashMap<>();
	paramters.put("type", "avro");
	paramters.put("hostname", "localhost");
	paramters.put("port", "41111");
	paramters.put("batch-size", "100");
	Context sinkContext = new Context(paramters);
	sparkAvroSink.configure(sinkContext);
	Configurables.configure(sparkAvroSink, sinkContext);
	sparkAvroSink.setChannel(sparkAvroChannel);

	sparkAvroChannel.start();
	sparkAvroSink.start();
}
 

@Test
public void testNullEmptyEvent() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("byteCapacity", "2000");
  parms.put("byteCapacityBufferPercentage", "20");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction tx = channel.getTransaction();
  tx.begin();
  //This line would cause a NPE without FLUME-1622.
  channel.put(EventBuilder.withBody(null));
  tx.commit();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody(new byte[0]));
  tx.commit();
  tx.close();


}
 

@Test
public void testNegativeCapacities() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "-3");
  parms.put("transactionCapacity", "-1");
  context.putAll(parms);
  Configurables.configure(channel, context);

  Assert.assertTrue(field("queue")
          .ofType(LinkedBlockingDeque.class)
          .in(channel).get()
          .remainingCapacity() > 0);

  Assert.assertTrue(field("transCapacity")
          .ofType(Integer.class)
          .in(channel).get() > 0);
}
 

@Test
public void testOptionalChannels() throws Exception {
  Context context = new Context();
  context.put(ReplicatingChannelSelector.CONFIG_OPTIONAL, "ch1");
  Configurables.configure(selector, context);
  List<Channel> channels = selector.getRequiredChannels(new MockEvent());
  Assert.assertNotNull(channels);
  Assert.assertEquals(3, channels.size());
  Assert.assertEquals("ch2", channels.get(0).getName());
  Assert.assertEquals("ch3", channels.get(1).getName());
  Assert.assertEquals("ch4", channels.get(2).getName());

  List<Channel> optCh = selector.getOptionalChannels(new MockEvent());
  Assert.assertEquals(1, optCh.size());
  Assert.assertEquals("ch1", optCh.get(0).getName());

}
 

@Test
public void testReconfigure() throws Exception {
  channel.start();
  Assert.assertTrue(channel.isOpen());
  Set<String> in = Sets.newHashSet();
  try {
    while(true) {
      in.addAll(putEvents(channel, "reconfig", 1, 1));
    }
  } catch (ChannelException e) {
    Assert.assertEquals("The channel has reached it's capacity. "
        + "This might be the result of a sink on the channel having too "
        + "low of batch size, a downstream system running slower than "
        + "normal, or that the channel capacity is just too low. [channel="
        + channel.getName()+"]", e.getMessage());
  }
  Configurables.configure(channel, createContext());
  Set<String> out = takeEvents(channel, 1, Integer.MAX_VALUE);
  compareInputAndOut(in, out);
}
 

@Test(expected=ChannelException.class)
public void testCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "3");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  // this should kill  it
  transaction.commit();
  Assert.fail();
}
 

@Ignore @Test
public void shouldIndexOneEvent() throws Exception {
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  Transaction tx = channel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody("event #1 or 1".getBytes());
  channel.put(event);
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();
  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

  assertMatchAllQuery(1, event);
  assertBodyQuery(1, event);
}
 

@Before
public void setUp() throws Exception {
  URL schemaUrl = getClass().getClassLoader().getResource("myrecord.avsc");
  Files.copy(Resources.newInputStreamSupplier(schemaUrl),
      new File("/tmp/myrecord.avsc"));

  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();
}
 

@Before
public void setUp() {
  source = new SpoolDirectorySource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  tmpDir = Files.createTempDir();
}
 

public void setUp(String compressionType, int compressionLevel) {
  if (sink != null) { throw new RuntimeException("double setup");}
  sink = new AvroSink();
  channel = new MemoryChannel();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));
  if (compressionType.equals("deflate")) {
    context.put("compression-type", compressionType);
    context.put("compression-level", Integer.toString(compressionLevel));
  }

  sink.setChannel(channel);

  Configurables.configure(sink, context);
  Configurables.configure(channel, context);
}
 

@Before
public void setUp() {
  source = new SyslogUDPSource(); //SyslogTcpSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  Context context = new Context();
  context.put("port", String.valueOf(TEST_SYSLOG_PORT));
  source.configure(context);
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) {
    try {
        channel = new NifiSinkSessionChannel(SUCCESS, FAILURE);
        channel.start();

        sink = SINK_FACTORY.create(context.getProperty(SOURCE_NAME).getValue(),
                context.getProperty(SINK_TYPE).getValue());
        sink.setChannel(channel);

        String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
        String agentName = context.getProperty(AGENT_NAME).getValue();
        String sinkName = context.getProperty(SOURCE_NAME).getValue();
        Configurables.configure(sink,
                getFlumeSinkContext(flumeConfig, agentName, sinkName));

        sink.start();
    } catch (Throwable th) {
        getLogger().error("Error creating sink", th);
        throw Throwables.propagate(th);
    }
}
 

@Test
public void testAppend() throws Exception {
  client = RpcClientFactory.getThriftInstance(props);
  Context context = new Context();
  channel.configure(context);
  configureSource();
  context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
  context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
  Configurables.configure(source, context);
  source.start();
  for(int i = 0; i < 30; i++) {
    client.append(EventBuilder.withBody(String.valueOf(i).getBytes()));
  }
  Transaction transaction = channel.getTransaction();
  transaction.begin();

  for (int i = 0; i < 30; i++) {
    Event event = channel.take();
    Assert.assertNotNull(event);
    Assert.assertEquals(String.valueOf(i), new String(event.getBody()));
  }
  transaction.commit();
  transaction.close();
}
 

@Test
@Ignore
public void testV1NoAuth() throws InterruptedException, IOException {
    Context context = new Context();
    context.put(CONF_TRAP_PORT, STMP_TRAP_PORT);
    context.put(CONF_SNMP_TRAP_VERSION, "V1");

    Configurables.configure(source, context);
    source.start();

    while (source.getSourceCounter().getEventAcceptedCount() < 1) {
        SNMPUtils.sendTrapV1(STMP_TRAP_PORT);
        Thread.sleep(10);
    }

    checkEventsChannel();
}
 

@Before
public void setUp() throws Exception {
  sink = new ThriftSink();
  channel = new MemoryChannel();
  hostname = "0.0.0.0";
  port = random.nextInt(50000) + 1024;
  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("request-timeout", String.valueOf(2000L));

  sink.setChannel(channel);

  Configurables.configure(sink, context);
  Configurables.configure(channel, context);
}
 

@Test
@Ignore
public void testV2NoAuth() throws InterruptedException, IOException {
    Context context = new Context();
    context.put(CONF_TRAP_PORT, STMP_TRAP_PORT);
    context.put(CONF_SNMP_TRAP_VERSION, "v2C");

    Configurables.configure(source, context);
    source.start();

    while (source.getSourceCounter().getEventAcceptedCount() < 1) {
        SNMPUtils.sendTrapV2(STMP_TRAP_PORT);
        Thread.sleep(10);
    }

    checkEventsChannel();
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) {
    try {
        source = SOURCE_FACTORY.create(
                context.getProperty(SOURCE_NAME).getValue(),
                context.getProperty(SOURCE_TYPE).getValue());

        String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
        String agentName = context.getProperty(AGENT_NAME).getValue();
        String sourceName = context.getProperty(SOURCE_NAME).getValue();
        Configurables.configure(source,
            getFlumeSourceContext(flumeConfig, agentName, sourceName));

        if (source instanceof PollableSource) {
            source.setChannelProcessor(new ChannelProcessor(
                new NifiChannelSelector(pollableSourceChannel)));
            source.start();
        }
    } catch (Throwable th) {
        getLogger().error("Error creating source", th);
        throw Throwables.propagate(th);
    }
}
 

@Test
public void testPutTake() throws InterruptedException, EventDeliveryException {
  Event event = EventBuilder.withBody("test event".getBytes());
  Context context = new Context();

  Configurables.configure(channel, context);

  Transaction transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  channel.put(event);
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  Event event2 = channel.take();
  Assert.assertEquals(event, event2);
  transaction.commit();
}
 

@BeforeMethod
public void setUp() throws Exception {
    if (sink != null) {
        throw new RuntimeException("double setup");
    }
    Context context = new Context();
    context.put("hostname", "127.0.0.1");
    context.put("port", "44445");
    context.put("batch-size", String.valueOf(2));
    context.put("connect-timeout", String.valueOf(2000L));
    context.put("request-timeout", String.valueOf(3000L));
    sink = new AvroSink();
    channel = new MemoryChannel();
    sink.setChannel(channel);
    Configurables.configure(sink, context);
    Configurables.configure(channel, context);

    mockSourceContext = mock(SourceContext.class);
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) {
    try {
        source = SOURCE_FACTORY.create(
                context.getProperty(SOURCE_NAME).getValue(),
                context.getProperty(SOURCE_TYPE).getValue());

        String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
        String agentName = context.getProperty(AGENT_NAME).getValue();
        String sourceName = context.getProperty(SOURCE_NAME).getValue();
        Configurables.configure(source,
            getFlumeSourceContext(flumeConfig, agentName, sourceName));

        if (source instanceof PollableSource) {
            source.setChannelProcessor(new ChannelProcessor(
                new NifiChannelSelector(pollableSourceChannel)));
            source.start();
        }
    } catch (Throwable th) {
        getLogger().error("Error creating source", th);
        throw Throwables.propagate(th);
    }
}
 

@Test(expected = FlumeException.class)
public void testMissingTable() throws Exception {
  deleteTable = false;
  ctx.put("batchSize", "2");
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  Assert.assertFalse(sink.isConfNull());
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 2);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 2; i++){
    for(int j = 0; j < 2; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(2, found);
  out = results[2];
  Assert.assertArrayEquals(Longs.toByteArray(2), out);
  sink.process();
  sink.stop();
}
 

@Test
public void testBatchProcessWithLifeCycle() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  int batchSize = 10;

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");
  context.put("batchSize", Integer.toString(batchSize));

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();

  for (long i = 0; i < 100; i++) {
    source.process();

    for (long j = batchSize; j > 0; j--) {
      Event event = channel.take();
      String expectedVal = String.valueOf(((i+1)*batchSize)-j);
      String resultedVal = new String(event.getBody());
      Assert.assertTrue("Expected " + expectedVal + " is not equals to " +
          resultedVal, expectedVal.equals(resultedVal));
    }
  }

  source.stop();
}
 

private Channel initChannel() {
    //Channel configuration
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");

    Channel channel = new MemoryChannel();
    channel.setName("memorychannel");
    Configurables.configure(channel, channelContext);
    return channel;
}
 

@Test
public void testOneEventWithDefaults() throws Exception {
  Map<String,String> ctxMap = new HashMap<String,String>();
  ctxMap.put("table", tableName);
  ctxMap.put("columnFamily", columnFamily);
  ctxMap.put("serializer",
          "org.apache.flume.sink.hbase.SimpleAsyncHbaseEventSerializer");
  ctxMap.put("keep-alive", "0");
  ctxMap.put("timeout", "10000");
  Context tmpctx = new Context();
  tmpctx.putAll(ctxMap);

  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, tmpctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, tmpctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = EventBuilder.withBody(
          Bytes.toBytes(valBase));
  channel.put(e);
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 1);
  byte[] out = results[0];
  Assert.assertArrayEquals(e.getBody(), out);
  out = results[1];
  Assert.assertArrayEquals(Longs.toByteArray(1), out);
}
 

@Test
public void shouldAllowCustomElasticSearchIndexRequestBuilderFactory()
    throws Exception {

  parameters.put(SERIALIZER,
      CustomElasticSearchIndexRequestBuilderFactory.class.getName());

  Configurables.configure(fixture, new Context(parameters));

  Channel channel = bindAndStartChannel(fixture);
  Transaction tx = channel.getTransaction();
  tx.begin();
  String body = "{ foo: \"bar\" }";
  Event event = EventBuilder.withBody(body.getBytes());
  channel.put(event);
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();

  assertEquals(fixture.getIndexName()+"-05_17_36_789",
      CustomElasticSearchIndexRequestBuilderFactory.actualIndexName);
  assertEquals(fixture.getIndexType(),
      CustomElasticSearchIndexRequestBuilderFactory.actualIndexType);
  assertArrayEquals(event.getBody(),
      CustomElasticSearchIndexRequestBuilderFactory.actualEventBody);
  assertTrue(CustomElasticSearchIndexRequestBuilderFactory.hasContext);
}
 

@Test
public void testKerbFileAccess() throws InterruptedException,
    LifecycleException, EventDeliveryException, IOException {
  LOG.debug("Starting testKerbFileAccess() ...");
  final String fileName = "FlumeData";
  final long rollCount = 5;
  final long batchSize = 2;
  String newPath = testPath + "/singleBucket";
  String kerbConfPrincipal = "user1/[email protected]";
  String kerbKeytab = "/usr/lib/flume/nonexistkeytabfile";

  //turn security on
  Configuration conf = new Configuration();
  conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);

  Context context = new Context();
  context.put("hdfs.path", newPath);
  context.put("hdfs.filePrefix", fileName);
  context.put("hdfs.rollCount", String.valueOf(rollCount));
  context.put("hdfs.batchSize", String.valueOf(batchSize));
  context.put("hdfs.kerberosPrincipal", kerbConfPrincipal);
  context.put("hdfs.kerberosKeytab", kerbKeytab);

  try {
    Configurables.configure(sink, context);
    Assert.fail("no exception thrown");
  } catch (IllegalArgumentException expected) {
    Assert.assertTrue(expected.getMessage().contains(
        "is nonexistent or can't read."));
  } finally {
    //turn security off
    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
        "simple");
    UserGroupInformation.setConfiguration(conf);
  }
}