Java Code Examples for org.apache.flume.channel.MemoryChannel#configure()

@Test
public void testCensor() {

  MemoryChannel memCh = new MemoryChannel();
  memCh.configure(new Context());
  memCh.start();

  ChannelSelector cs = new ReplicatingChannelSelector();
  cs.setChannels(Lists.<Channel>newArrayList(memCh));
  ChannelProcessor cp = new ChannelProcessor(cs);

  // source config
  Map<String, String> cfgMap = Maps.newHashMap();
  cfgMap.put("interceptors", "a");
  String builderClass = CensoringInterceptor.Builder.class.getName();
  cfgMap.put("interceptors.a.type", builderClass);
  Context ctx = new Context(cfgMap);

  // setup
  cp.configure(ctx);
  cp.initialize();

  Map<String, String> headers = Maps.newHashMap();
  String badWord = "scribe";
  headers.put("Bad-Words", badWord);
  Event event1 = EventBuilder.withBody("test", Charsets.UTF_8, headers);
  Assert.assertEquals(badWord, event1.getHeaders().get("Bad-Words"));
  cp.processEvent(event1);

  Transaction tx = memCh.getTransaction();
  tx.begin();

  Event event1a = memCh.take();
  Assert.assertNull(event1a.getHeaders().get("Bad-Words"));

  tx.commit();
  tx.close();

  // cleanup / shutdown
  cp.close();
  memCh.stop();
}
 

/**
 * Test that different charsets are parsed by different ports correctly.
 */
@Test
public void testPortCharsetHandling() throws UnknownHostException, Exception {

  ///////////////////////////////////////////////////////
  // port setup

  InetAddress localAddr = InetAddress.getLocalHost();
  DefaultIoSessionDataStructureFactory dsFactory =
      new DefaultIoSessionDataStructureFactory();


  // one faker on port 10001
  int port1 = 10001;
  NioSession session1 = mock(NioSession.class);
  session1.setAttributeMap(dsFactory.getAttributeMap(session1));
  SocketAddress sockAddr1 = new InetSocketAddress(localAddr, port1);
  when(session1.getLocalAddress()).thenReturn(sockAddr1);

  // another faker on port 10002
  int port2 = 10002;
  NioSession session2 = mock(NioSession.class);
  session2.setAttributeMap(dsFactory.getAttributeMap(session2));
  SocketAddress sockAddr2 = new InetSocketAddress(localAddr, port2);
  when(session2.getLocalAddress()).thenReturn(sockAddr2);

  // set up expected charsets per port
  ConcurrentMap<Integer, ThreadSafeDecoder> portCharsets =
      new ConcurrentHashMap<Integer, ThreadSafeDecoder>();
  portCharsets.put(port1, new ThreadSafeDecoder(Charsets.ISO_8859_1));
  portCharsets.put(port2, new ThreadSafeDecoder(Charsets.UTF_8));

  ///////////////////////////////////////////////////////
  // channel / source setup

  // set up channel to receive events
  MemoryChannel chan = new MemoryChannel();
  chan.configure(new Context());
  chan.start();
  ReplicatingChannelSelector sel = new ReplicatingChannelSelector();
  sel.setChannels(Lists.<Channel>newArrayList(chan));
  ChannelProcessor chanProc = new ChannelProcessor(sel);

  // defaults to UTF-8
  MultiportSyslogHandler handler = new MultiportSyslogHandler(
      1000, 10, chanProc, new SourceCounter("test"), "port",
      new ThreadSafeDecoder(Charsets.UTF_8), portCharsets);

  // initialize buffers
  handler.sessionCreated(session1);
  handler.sessionCreated(session2);

  ///////////////////////////////////////////////////////
  // event setup

  // Create events of varying charsets.
  String header = "<10>2012-08-17T02:14:00-07:00 192.168.1.110 ";

  // These chars encode under ISO-8859-1 as illegal bytes under UTF-8.
  String dangerousChars = "þÿÀÁ";

  ///////////////////////////////////////////////////////
  // encode and send them through the message handler
  String msg;
  IoBuffer buf;
  Event evt;

  // valid ISO-8859-1 on the right (ISO-8859-1) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.ISO_8859_1));
  handler.messageReceived(session1, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertNull(evt.getHeaders().get(SyslogUtils.EVENT_STATUS));

  // valid ISO-8859-1 on the wrong (UTF-8) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.ISO_8859_1));
  handler.messageReceived(session2, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertEquals("Expected invalid event due to character encoding",
      SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(),
      evt.getHeaders().get(SyslogUtils.EVENT_STATUS));

  // valid UTF-8 on the right (UTF-8) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.UTF_8));
  handler.messageReceived(session2, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertNull(evt.getHeaders().get(SyslogUtils.EVENT_STATUS));
}
 

/**
 * This is a very basic test that writes one event to HDFS and reads it back.
 */
@Test
public void simpleHDFSTest() throws EventDeliveryException, IOException {
  cluster = new MiniDFSCluster(new Configuration(), 1, true, null);
  cluster.waitActive();

  String outputDir = "/flume/simpleHDFSTest";
  Path outputDirPath = new Path(outputDir);

  logger.info("Running test with output dir: {}", outputDir);

  FileSystem fs = cluster.getFileSystem();
  // ensure output directory is empty
  if (fs.exists(outputDirPath)) {
    fs.delete(outputDirPath, true);
  }

  String nnURL = getNameNodeURL(cluster);
  logger.info("Namenode address: {}", nnURL);

  Context chanCtx = new Context();
  MemoryChannel channel = new MemoryChannel();
  channel.setName("simpleHDFSTest-mem-chan");
  channel.configure(chanCtx);
  channel.start();

  Context sinkCtx = new Context();
  sinkCtx.put("hdfs.path", nnURL + outputDir);
  sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
  sinkCtx.put("hdfs.batchSize", Integer.toString(1));

  HDFSEventSink sink = new HDFSEventSink();
  sink.setName("simpleHDFSTest-hdfs-sink");
  sink.configure(sinkCtx);
  sink.setChannel(channel);
  sink.start();

  // create an event
  String EVENT_BODY = "yarg!";
  channel.getTransaction().begin();
  try {
    channel.put(EventBuilder.withBody(EVENT_BODY, Charsets.UTF_8));
    channel.getTransaction().commit();
  } finally {
    channel.getTransaction().close();
  }

  // store event to HDFS
  sink.process();

  // shut down flume
  sink.stop();
  channel.stop();

  // verify that it's in HDFS and that its content is what we say it should be
  FileStatus[] statuses = fs.listStatus(outputDirPath);
  Assert.assertNotNull("No files found written to HDFS", statuses);
  Assert.assertEquals("Only one file expected", 1, statuses.length);

  for (FileStatus status : statuses) {
    Path filePath = status.getPath();
    logger.info("Found file on DFS: {}", filePath);
    FSDataInputStream stream = fs.open(filePath);
    BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
    String line = reader.readLine();
    logger.info("First line in file {}: {}", filePath, line);
    Assert.assertEquals(EVENT_BODY, line);
  }

  if (!KEEP_DATA) {
    fs.delete(outputDirPath, true);
  }

  cluster.shutdown();
  cluster = null;
}
 

/**
 * Writes two events in GZIP-compressed serialize.
 */
@Test
public void simpleHDFSGZipCompressedTest() throws EventDeliveryException, IOException {
  cluster = new MiniDFSCluster(new Configuration(), 1, true, null);
  cluster.waitActive();

  String outputDir = "/flume/simpleHDFSGZipCompressedTest";
  Path outputDirPath = new Path(outputDir);

  logger.info("Running test with output dir: {}", outputDir);

  FileSystem fs = cluster.getFileSystem();
  // ensure output directory is empty
  if (fs.exists(outputDirPath)) {
    fs.delete(outputDirPath, true);
  }

  String nnURL = getNameNodeURL(cluster);
  logger.info("Namenode address: {}", nnURL);

  Context chanCtx = new Context();
  MemoryChannel channel = new MemoryChannel();
  channel.setName("simpleHDFSTest-mem-chan");
  channel.configure(chanCtx);
  channel.start();

  Context sinkCtx = new Context();
  sinkCtx.put("hdfs.path", nnURL + outputDir);
  sinkCtx.put("hdfs.fileType", HDFSWriterFactory.CompStreamType);
  sinkCtx.put("hdfs.batchSize", Integer.toString(1));
  sinkCtx.put("hdfs.codeC", "gzip");

  HDFSEventSink sink = new HDFSEventSink();
  sink.setName("simpleHDFSTest-hdfs-sink");
  sink.configure(sinkCtx);
  sink.setChannel(channel);
  sink.start();

  // create an event
  String EVENT_BODY_1 = "yarg1";
  String EVENT_BODY_2 = "yarg2";
  channel.getTransaction().begin();
  try {
    channel.put(EventBuilder.withBody(EVENT_BODY_1, Charsets.UTF_8));
    channel.put(EventBuilder.withBody(EVENT_BODY_2, Charsets.UTF_8));
    channel.getTransaction().commit();
  } finally {
    channel.getTransaction().close();
  }

  // store event to HDFS
  sink.process();

  // shut down flume
  sink.stop();
  channel.stop();

  // verify that it's in HDFS and that its content is what we say it should be
  FileStatus[] statuses = fs.listStatus(outputDirPath);
  Assert.assertNotNull("No files found written to HDFS", statuses);
  Assert.assertEquals("Only one file expected", 1, statuses.length);

  for (FileStatus status : statuses) {
    Path filePath = status.getPath();
    logger.info("Found file on DFS: {}", filePath);
    FSDataInputStream stream = fs.open(filePath);
    BufferedReader reader = new BufferedReader(new InputStreamReader(
        new GZIPInputStream(stream)));
    String line = reader.readLine();
    logger.info("First line in file {}: {}", filePath, line);
    Assert.assertEquals(EVENT_BODY_1, line);

    // The rest of this test is commented-out (will fail) for 2 reasons:
    //
    // (1) At the time of this writing, Hadoop has a bug which causes the
    // non-native gzip implementation to create invalid gzip files when
    // finish() and resetState() are called. See HADOOP-8522.
    //
    // (2) Even if HADOOP-8522 is fixed, the JDK GZipInputStream is unable
    // to read multi-member (concatenated) gzip files. See this Sun bug:
    // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4691425
    //
    //line = reader.readLine();
    //logger.info("Second line in file {}: {}", filePath, line);
    //Assert.assertEquals(EVENT_BODY_2, line);
  }

  if (!KEEP_DATA) {
    fs.delete(outputDirPath, true);
  }

  cluster.shutdown();
  cluster = null;
}