Java Code Examples for org.apache.flume.ChannelSelector#setChannels()

@Before
public void setUp() throws Exception {
  port = NetworkUtils.getRandomPort();
  source = new ThriftSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<>();
  channels.add(ch);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);
  source.setChannelProcessor(new ChannelProcessor(rcs));
  source.start();
}
 

@Before
public void setUp() {
  source = new SpoolDirectorySource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  tmpDir = Files.createTempDir();
}
 

@Before
public void setUp() {
  source = new SyslogUDPSource(); //SyslogTcpSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  Context context = new Context();
  context.put("port", String.valueOf(TEST_SYSLOG_PORT));
  source.configure(context);
}
 

/**
 * Ensure that we see the original NPE from the PreConditions check instead
 * of an auto-generated NPE, which could be masking something else.
 */
@Test
public void testNullFromGetTransaction() {
  // channel which returns null from getTransaction()
  Channel ch = mock(Channel.class);
  when(ch.getTransaction()).thenReturn(null);

  ChannelSelector sel = new ReplicatingChannelSelector();
  sel.setChannels(Lists.newArrayList(ch));
  ChannelProcessor proc = new ChannelProcessor(sel);

  List<Event> events = Lists.newArrayList();
  events.add(EventBuilder.withBody("event 1", Charsets.UTF_8));

  boolean threw = false;
  try {
    proc.processEventBatch(events);
  } catch (NullPointerException ex) {
    threw = true;
    Assert.assertNotNull("NPE must be manually thrown", ex.getMessage());
  }
  Assert.assertTrue("Must throw NPE", threw);
}
 

@Before
public void setUp() throws Exception {
  URL schemaUrl = getClass().getClassLoader().getResource("myrecord.avsc");
  Files.copy(Resources.newInputStreamSupplier(schemaUrl),
      new File("/tmp/myrecord.avsc"));

  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();
}
 

@Before
public void setUp() {
  logger.info("Running setup");

  channel = new MemoryChannel();
  source = new NetcatSource();

  Context context = new Context();

  Configurables.configure(channel, context);
  List<Channel> channels = Lists.newArrayList(channel);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

public static ChannelSelector create(List<Channel> channels,
    Map<String, String> config) {

  ChannelSelector selector = getSelectorForType(config.get(
      BasicConfigurationConstants.CONFIG_TYPE));

  selector.setChannels(channels);

  Context context = new Context();
  context.putAll(config);

  Configurables.configure(selector, context);
  return selector;
}
 

@Before
public void setUp() {
    source = new SNMPSource();
    channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);

    source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

@Before
public void setUp() {
    source = new RedisSource();
    channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);

    source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

@Before
public void setUp() {
  source = new AvroSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

@BeforeClass
public static void setUpClass() throws Exception {
  selectedPort = findFreePort();

  source = new HTTPSource();
  channel = new MemoryChannel();

  Context ctx = new Context();
  ctx.put("capacity", "100");
  Configurables.configure(channel, ctx);

  List<Channel> channels = new ArrayList<Channel>(1);
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  channel.start();
  Context context = new Context();

  context.put("port", String.valueOf(selectedPort));
  context.put("host", "0.0.0.0");

  Configurables.configure(source, context);
  source.start();
}
 

@Before
public void setUp() throws Exception {
    eventSource = new AvroSource();
    channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    avroLogger = (Logger) LogManager.getLogger("avrologger");
    /*
     * Clear out all other appenders associated with this logger to ensure
     * we're only hitting the Avro appender.
     */
    removeAppenders(avroLogger);
    final Context context = new Context();
    testPort = String.valueOf(AvailablePortFinder.getNextAvailable());
    context.put("port", testPort);
    context.put("bind", "0.0.0.0");
    Configurables.configure(eventSource, context);

    final List<Channel> channels = new ArrayList<>();
    channels.add(channel);

    final ChannelSelector cs = new ReplicatingChannelSelector();
    cs.setChannels(channels);

    eventSource.setChannelProcessor(new ChannelProcessor(cs));

    eventSource.start();

    Assert.assertTrue("Reached start or error", LifecycleController
            .waitForOneOf(eventSource, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
            eventSource.getLifecycleState());
}
 

@Test
public void testProcess() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  for (long i = 0; i < 100; i++) {
    source.process();
    Event event = channel.take();

    Assert.assertArrayEquals(String.valueOf(i).getBytes(),
        new String(event.getBody()).getBytes());
  }
}
 

@Test
public void testBatchProcessWithLifeCycle() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  int batchSize = 10;

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");
  context.put("batchSize", Integer.toString(batchSize));

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();

  for (long i = 0; i < 100; i++) {
    source.process();

    for (long j = batchSize; j > 0; j--) {
      Event event = channel.take();
      String expectedVal = String.valueOf(((i+1)*batchSize)-j);
      String resultedVal = new String(event.getBody());
      Assert.assertTrue("Expected " + expectedVal + " is not equals to " +
          resultedVal, expectedVal.equals(resultedVal));
    }
  }

  source.stop();
}
 

@Before
public void setUp() {
  source = new ThriftLegacySource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

@SuppressWarnings("unused")
private void createAvroSourceWithLocalFileRollingSink() {
	channel = new MemoryChannel();
	String channelName = "AvroSourceMemoryChannel-" + UUID.randomUUID();
	channel.setName(channelName);

	sink = new RollingFileSink();
	sink.setName("RollingFileSink-" + UUID.randomUUID());
	Map<String, String> paramters = new HashMap<>();
	paramters.put("type", "file_roll");
	paramters.put("sink.directory", "target/flumefilelog");
	Context sinkContext = new Context(paramters);
	sink.configure(sinkContext);
	Configurables.configure(channel, sinkContext);
	sink.setChannel(channel);

	final Map<String, String> properties = new HashMap<String, String>();
	properties.put("type", "avro");
	properties.put("bind", "localhost");
	properties.put("port", "44444");
	properties.put("selector.type", "multiplexing");
	properties.put("selector.header", "State");
	properties.put("selector.mapping.VIEWED", channelName);
	properties.put("selector.mapping.default", channelName);

	avroSource = new AvroSource();
	avroSource.setName("AvroSource-" + UUID.randomUUID());
	Context sourceContext = new Context(properties);
	avroSource.configure(sourceContext);
	ChannelSelector selector = new MultiplexingChannelSelector();
	List<Channel> channels = new ArrayList<>();
	channels.add(channel);
	selector.setChannels(channels);
	final Map<String, String> selectorProperties = new HashMap<String, String>();
	properties.put("default", channelName);
	Context selectorContext = new Context(selectorProperties);
	selector.configure(selectorContext);
	ChannelProcessor cp = new ChannelProcessor(selector);
	avroSource.setChannelProcessor(cp);

	sink.start();
	channel.start();
	avroSource.start();
}
 

@Test
public void testBasic() throws Exception {
    System.out.println(getTestTraceHead("[TwitterSourceTest.basic]")
            + "-------- Start source.");
    Context context = new Context();
    context.put("consumerKey", consumerKey);
    context.put("consumerSecret", consumerSecret);
    context.put("accessToken", accessToken);
    context.put("accessTokenSecret", accessTokenSecret);
    context.put("maxBatchDurationMillis", "1000");

    TwitterSource source = new TwitterSource();
    source.configure(context);

    Map<String, String> channelContext = new HashMap();
    channelContext.put("capacity", "1000000");
    channelContext.put("keep-alive", "0"); // for faster tests
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context(channelContext));

    Sink sink = new LoggerSink();
    sink.setChannel(channel);
    sink.start();
    DefaultSinkProcessor proc = new DefaultSinkProcessor();
    proc.setSinks(Collections.singletonList(sink));
    SinkRunner sinkRunner = new SinkRunner(proc);
    sinkRunner.start();

    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(Collections.singletonList(channel));
    ChannelProcessor chp = new ChannelProcessor(rcs);
    source.setChannelProcessor(chp);

    try {
        source.start();

        Thread.sleep(500);
        source.stop();
        System.out.println(getTestTraceHead("[TwitterSourceTest.basic]")
                + "-  OK  - Twitter source started properly.");
    } catch (AssertionError e) {
        System.out.println(getTestTraceHead("[TwitterSourceTest.basic]")
                + "- FAIL - Twitter source could not start.");
        throw e;
    } // try catch
    sinkRunner.stop();
    sink.stop();
}
 

private void createAvroSourceWithSelectorHDFSAndESSinks() {
	Channel ESChannel = flumeESSinkService.getChannel();
	Channel HDFSChannel = flumeHDFSSinkService.getChannel();
	Channel HbaseChannel = flumeHbaseSinkService.getChannel();

	final Map<String, String> properties = new HashMap<String, String>();
	properties.put("type", "avro");
	properties.put("bind", "localhost");
	properties.put("port", "44444");

	avroSource = new AvroSource();
	avroSource.setName("AvroSource-" + UUID.randomUUID());
	Context sourceContext = new Context(properties);
	avroSource.configure(sourceContext);
	ChannelSelector selector = new MultiplexingChannelSelector();
	List<Channel> channels = new ArrayList<>();
	channels.add(ESChannel);
	channels.add(HDFSChannel);
	channels.add(sparkAvroChannel);
	channels.add(HbaseChannel);
	selector.setChannels(channels);
	final Map<String, String> selectorProperties = new HashMap<String, String>();
	selectorProperties.put("type", "multiplexing");
	selectorProperties.put("header", "State");
	// selectorProperties.put("mapping.VIEWED", HDFSChannel.getName() + " "
	// + ESChannel.getName());
	// selectorProperties.put("mapping.FAVOURITE", HDFSChannel.getName() +
	// " "
	// + ESChannel.getName());
	// selectorProperties.put("default", HDFSChannel.getName());
	// In case spark avro sink is used.
	selectorProperties.put("mapping.VIEWED", HDFSChannel.getName() + " "
			+ ESChannel.getName() + " " + sparkAvroChannel.getName() + " "
			+ HbaseChannel.getName());
	selectorProperties.put("mapping.FAVOURITE", HDFSChannel.getName() + " "
			+ ESChannel.getName() + " " + sparkAvroChannel.getName() + " "
			+ HbaseChannel.getName());
	selectorProperties.put("default", HDFSChannel.getName() + " "
			+ sparkAvroChannel.getName() + " " + HbaseChannel.getName());
	Context selectorContext = new Context(selectorProperties);
	selector.configure(selectorContext);
	ChannelProcessor cp = new ChannelProcessor(selector);
	avroSource.setChannelProcessor(cp);

	avroSource.start();
}
 

@Test
public void testCensor() {

  MemoryChannel memCh = new MemoryChannel();
  memCh.configure(new Context());
  memCh.start();

  ChannelSelector cs = new ReplicatingChannelSelector();
  cs.setChannels(Lists.<Channel>newArrayList(memCh));
  ChannelProcessor cp = new ChannelProcessor(cs);

  // source config
  Map<String, String> cfgMap = Maps.newHashMap();
  cfgMap.put("interceptors", "a");
  String builderClass = CensoringInterceptor.Builder.class.getName();
  cfgMap.put("interceptors.a.type", builderClass);
  Context ctx = new Context(cfgMap);

  // setup
  cp.configure(ctx);
  cp.initialize();

  Map<String, String> headers = Maps.newHashMap();
  String badWord = "scribe";
  headers.put("Bad-Words", badWord);
  Event event1 = EventBuilder.withBody("test", Charsets.UTF_8, headers);
  Assert.assertEquals(badWord, event1.getHeaders().get("Bad-Words"));
  cp.processEvent(event1);

  Transaction tx = memCh.getTransaction();
  tx.begin();

  Event event1a = memCh.take();
  Assert.assertNull(event1a.getHeaders().get("Bad-Words"));

  tx.commit();
  tx.close();

  // cleanup / shutdown
  cp.close();
  memCh.stop();
}
 

@Test
	public void testProcess() throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
		Channel channel = new MemoryChannel();
		Context context = new Context();

		String resource = "src/test/resources/server.log";
		assertNotNull(resource);

		int expectedNrOfEvents = 0;

		BufferedReader br = new BufferedReader(new FileReader(resource));
		String line;
		while ((line = br.readLine()) != null) {
			if (line.endsWith("|#]")) {
				expectedNrOfEvents++;
			}
		}

		context.put("command", format("cat %s", resource));
		context.put("event.terminator", "|#]");
		context.put("keep-alive", "1");
		context.put("capacity", "100000");
		context.put("transactionCapacity", "100000");
		Configurables.configure(source, context);
		Configurables.configure(channel, context);

		ChannelSelector rcs = new ReplicatingChannelSelector();
		rcs.setChannels(Lists.newArrayList(channel));

		source.setChannelProcessor(new ChannelProcessor(rcs));

		source.start();
		Transaction transaction = channel.getTransaction();

		transaction.begin();
		Event event;

		int actualNrOfEvents = 0;

		FileOutputStream outputStream = new FileOutputStream("/tmp/flume-execsource." + Thread.currentThread().getId());

		while ((event = channel.take()) != null) {
			outputStream.write(event.getBody());
			outputStream.write('\n');
			outputStream.write('\n');
			actualNrOfEvents++;
		}

		outputStream.close();
		transaction.commit();
		transaction.close();

		source.stop();

		File actualFile = new File("/tmp/flume-execsource." + Thread.currentThread().getId());
		File expectedFile = new File(resource);

		assertEquals(expectedNrOfEvents, actualNrOfEvents);

        // This doesn't work anymore since we implemented the different event separator § instead of \n
//		assertEquals(FileUtils.checksumCRC32(expectedFile), FileUtils.checksumCRC32(actualFile));

		FileUtils.forceDelete(actualFile);
	}