org.apache.flume.instrumentation.SourceCounter Java Examples

@Override
public void configure(Context context) {
    // Only the queue name does not have a default value
    Configurables.ensureRequiredNonNull(context, QUEUE_KEY);

    // Assign all of the configured values
    hostname = context.getString(HOST_KEY, ConnectionFactory.DEFAULT_HOST);
    port = context.getInteger(PORT_KEY, ConnectionFactory.DEFAULT_AMQP_PORT);
    enableSSL = context.getBoolean(SSL_KEY, false);
    virtualHost = context.getString(VHOST_KEY, ConnectionFactory.DEFAULT_VHOST);
    username = context.getString(USER_KEY, ConnectionFactory.DEFAULT_USER);
    password = context.getString(PASSWORD_KEY, ConnectionFactory.DEFAULT_PASS);
    queue = context.getString(QUEUE_KEY, null);
    exchange = context.getString(EXCHANGE_KEY, null);
    autoAck = context.getBoolean(AUTOACK_KEY, false);
    requeuing = context.getBoolean(REQUEUING, false);
    prefetchCount = context.getInteger(PREFETCH_COUNT_KEY, 0);
    timeout = context.getInteger(TIMEOUT_KEY, -1);
    consumerThreads = context.getInteger(THREAD_COUNT_KEY, 1);

    // Ensure that Flume can connect to RabbitMQ
    testRabbitMQConnection();

    // Create and configure the counters
    sourceCounter = new SourceCounter(getName());
    counterGroup = new CounterGroup();
    counterGroup.setName(getName());
}
 

@Override
public void configure(Context context) {
    // Only the queue name does not have a default value
    Configurables.ensureRequiredNonNull(context, QUEUE_KEY);

    // Assign all of the configured values
    hostname = context.getString(HOST_KEY, ConnectionFactory.DEFAULT_HOST);
    port = context.getInteger(PORT_KEY, ConnectionFactory.DEFAULT_AMQP_PORT);
    enableSSL = context.getBoolean(SSL_KEY, false);
    virtualHost = context.getString(VHOST_KEY, ConnectionFactory.DEFAULT_VHOST);
    username = context.getString(USER_KEY, ConnectionFactory.DEFAULT_USER);
    password = context.getString(PASSWORD_KEY, ConnectionFactory.DEFAULT_PASS);
    queue = context.getString(QUEUE_KEY, null);
    exchange = context.getString(EXCHANGE_KEY, null);
    autoAck = context.getBoolean(AUTOACK_KEY, false);
    requeuing = context.getBoolean(REQUEUING, false);
    prefetchCount = context.getInteger(PREFETCH_COUNT_KEY, 0);
    timeout = context.getInteger(TIMEOUT_KEY, -1);
    consumerThreads = context.getInteger(THREAD_COUNT_KEY, 1);

    // Ensure that Flume can connect to RabbitMQ
    testRabbitMQConnection();

    // Create and configure the counters
    sourceCounter = new SourceCounter(getName());
    counterGroup = new CounterGroup();
    counterGroup.setName(getName());
}
 

public ExecRunnable(String shell, String command, ChannelProcessor channelProcessor,
    SourceCounter sourceCounter, boolean restart, long restartThrottle,
    boolean logStderr, int bufferCount, long batchTimeout, Charset charset) {
  this.command = command;
  this.channelProcessor = channelProcessor;
  this.sourceCounter = sourceCounter;
  this.restartThrottle = restartThrottle;
  this.bufferCount = bufferCount;
  this.batchTimeout = batchTimeout;
  this.restart = restart;
  this.logStderr = logStderr;
  this.charset = charset;
  this.shell = shell;
}
 

public MultiportSyslogHandler(int maxEventSize, int batchSize,
    ChannelProcessor cp, SourceCounter ctr, String portHeader,
    ThreadSafeDecoder defaultDecoder,
    ConcurrentMap<Integer, ThreadSafeDecoder> portCharsets) {
  channelProcessor = cp;
  sourceCounter = ctr;
  this.maxEventSize = maxEventSize;
  this.batchSize = batchSize;
  this.portHeader = portHeader;
  this.defaultDecoder = defaultDecoder;
  this.portCharsets = portCharsets;
  syslogParser = new SyslogParser();
  lineSplitter = new LineSplitter(maxEventSize);
}
 

@VisibleForTesting
protected SourceCounter getSourceCounter() {
    return sourceCounter;
}
 

public SpoolFileRunnable(ReliableSpoolFileEventReader reader, SourceCounter sourceCounter) {
    this.reader = reader;
    this.sourceCounter = sourceCounter;
}
 

@Override
public synchronized void doStart() {
    log.info("start source of flume ...");
    this.counter = new SourceCounter("flume-source");
    this.counter.start();
}
 

public SpoolDirectoryTailFileRunnable(ReliableSpoolDirectoryTailFileEventReader reader, SourceCounter sourceCounter) {
	this.reader = reader;
	this.sourceCounter = sourceCounter;
}
 

public Consumer setSourceCounter(SourceCounter sourceCounter) {
    this.sourceCounter = sourceCounter;
    return this;
}
 

public Consumer setSourceCounter(SourceCounter sourceCounter) {
    this.sourceCounter = sourceCounter;
    return this;
}
 

public SpoolDirectoryRunnable(ReliableSpoolingFileEventReader reader,
    SourceCounter sourceCounter) {
  this.reader = reader;
  this.sourceCounter = sourceCounter;
}
 

/**
 * Test that different charsets are parsed by different ports correctly.
 */
@Test
public void testPortCharsetHandling() throws UnknownHostException, Exception {

  ///////////////////////////////////////////////////////
  // port setup

  InetAddress localAddr = InetAddress.getLocalHost();
  DefaultIoSessionDataStructureFactory dsFactory =
      new DefaultIoSessionDataStructureFactory();


  // one faker on port 10001
  int port1 = 10001;
  NioSession session1 = mock(NioSession.class);
  session1.setAttributeMap(dsFactory.getAttributeMap(session1));
  SocketAddress sockAddr1 = new InetSocketAddress(localAddr, port1);
  when(session1.getLocalAddress()).thenReturn(sockAddr1);

  // another faker on port 10002
  int port2 = 10002;
  NioSession session2 = mock(NioSession.class);
  session2.setAttributeMap(dsFactory.getAttributeMap(session2));
  SocketAddress sockAddr2 = new InetSocketAddress(localAddr, port2);
  when(session2.getLocalAddress()).thenReturn(sockAddr2);

  // set up expected charsets per port
  ConcurrentMap<Integer, ThreadSafeDecoder> portCharsets =
      new ConcurrentHashMap<Integer, ThreadSafeDecoder>();
  portCharsets.put(port1, new ThreadSafeDecoder(Charsets.ISO_8859_1));
  portCharsets.put(port2, new ThreadSafeDecoder(Charsets.UTF_8));

  ///////////////////////////////////////////////////////
  // channel / source setup

  // set up channel to receive events
  MemoryChannel chan = new MemoryChannel();
  chan.configure(new Context());
  chan.start();
  ReplicatingChannelSelector sel = new ReplicatingChannelSelector();
  sel.setChannels(Lists.<Channel>newArrayList(chan));
  ChannelProcessor chanProc = new ChannelProcessor(sel);

  // defaults to UTF-8
  MultiportSyslogHandler handler = new MultiportSyslogHandler(
      1000, 10, chanProc, new SourceCounter("test"), "port",
      new ThreadSafeDecoder(Charsets.UTF_8), portCharsets);

  // initialize buffers
  handler.sessionCreated(session1);
  handler.sessionCreated(session2);

  ///////////////////////////////////////////////////////
  // event setup

  // Create events of varying charsets.
  String header = "<10>2012-08-17T02:14:00-07:00 192.168.1.110 ";

  // These chars encode under ISO-8859-1 as illegal bytes under UTF-8.
  String dangerousChars = "þÿÀÁ";

  ///////////////////////////////////////////////////////
  // encode and send them through the message handler
  String msg;
  IoBuffer buf;
  Event evt;

  // valid ISO-8859-1 on the right (ISO-8859-1) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.ISO_8859_1));
  handler.messageReceived(session1, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertNull(evt.getHeaders().get(SyslogUtils.EVENT_STATUS));

  // valid ISO-8859-1 on the wrong (UTF-8) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.ISO_8859_1));
  handler.messageReceived(session2, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertEquals("Expected invalid event due to character encoding",
      SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(),
      evt.getHeaders().get(SyslogUtils.EVENT_STATUS));

  // valid UTF-8 on the right (UTF-8) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.UTF_8));
  handler.messageReceived(session2, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertNull(evt.getHeaders().get(SyslogUtils.EVENT_STATUS));
}
 

@VisibleForTesting
protected SourceCounter getSourceCounter() {
    return sourceCounter;
}
 

@VisibleForTesting
protected SourceCounter getSourceCounter() {
    return sourceCounter;
}