org.apache.flume.FlumeException Java Examples

/**
 * Start this server, causing it to poll JMX at the configured frequency.
 */
@Override
public void start() {
  try {
    socket = new DatagramSocket();
    hostname = InetAddress.getLocalHost().getHostName();
  } catch (SocketException ex) {
    logger.error("Could not create socket for metrics collection.");
    throw new FlumeException(
            "Could not create socket for metrics collection.", ex);
  } catch (Exception ex2) {
    logger.warn("Unknown error occured", ex2);
  }
  for (HostInfo host : hosts) {
    addresses.add(new InetSocketAddress(
            host.getHostName(), host.getPortNumber()));
  }
  collectorRunnable.server = this;
  if (service.isShutdown() || service.isTerminated()) {
    service = Executors.newSingleThreadScheduledExecutor();
  }
  service.scheduleWithFixedDelay(collectorRunnable, 0,
          pollFrequency, TimeUnit.SECONDS);
}
 

@Override
public void start() {
    try {
        LOGGER.info("Starting Kafka Channel: " + getName());
        producer = new Producer<String, byte[]>(new ProducerConfig(kafkaConf));
        // We always have just one topic being read by one thread
        LOGGER.info("Topic = " + topic.get());
        topicCountMap.put(topic.get(), 1);
        counter.start();
        super.start();
    } catch (Exception e) {
        LOGGER.error("Could not start producer");
        throw new FlumeException("Unable to create Kafka Connections. " +
                "Check whether Kafka Brokers are up and that the " +
                "Flume agent can connect to it.", e);
    }
}
 

/**
 * Create / reconnect Flume RPC client
 */
private void startClient() {
  // If current client is inactive, close it
  if (flumeClient != null && !flumeClient.isActive()) {
    flumeClient.close();
    flumeClient = null;
  }
  // Create client if needed
  if (flumeClient == null) {
    try {
      flumeClient = RpcClientFactory.getDefaultInstance(flumeHostName, flumePort, maxSpanBatchSize);
    } catch (FlumeException e) {
      LOG.warn("Failed to create Flume RPC Client. " + e.getMessage());
    }
  }
}
 

private Method reflectHflushOrSync(FSDataOutputStream os) {
  Method m = null;
  if(os != null) {
    Class<?> fsDataOutputStreamClass = os.getClass();
    try {
      m = fsDataOutputStreamClass.getMethod("hflush");
    } catch (NoSuchMethodException ex) {
      logger.debug("HFlush not found. Will use sync() instead");
      try {
        m = fsDataOutputStreamClass.getMethod("sync");
      } catch (Exception ex1) {
        String msg = "Neither hflush not sync were found. That seems to be " +
          "a problem!";
        logger.error(msg);
        throw new FlumeException(msg, ex1);
      }
    }
  }
  return m;
}
 

/**
 * Started the agent. Can only be called after a successful call to
 * configure().
 *
 * @throws FlumeException if a component cannot be started
 * @throws IllegalStateException if the agent has not been configured or is
 * already started
 */
public void start()
    throws FlumeException {
  if(state == State.STARTED) {
    throw new IllegalStateException("Cannot be started while started");
  } else if(state == State.NEW) {
    throw new IllegalStateException("Cannot be started before being " +
        "configured");
  }
  doStart();
  Source source = Preconditions.checkNotNull(sourceRunner.getSource(),
      "Source runner returned null source");
  if(source instanceof EmbeddedSource) {
    embeddedSource = (EmbeddedSource)source;
  } else {
    throw new IllegalStateException("Unknown source type: " + source.
        getClass().getName());
  }
  state = State.STARTED;
}
 

@Override
protected void doConfigure(Context context) throws FlumeException {
    ensureRequiredNonNull(context, HOST_KEY, PORT_KEY, PATH_KEY);

    this.host = context.getString(HOST_KEY);
    this.port = context.getInteger(PORT_KEY);
    this.path = context.getString(PATH_KEY);
    this.enableSsl = context.getBoolean(SSL_KEY, false);
    this.keystore = context.getString(KEYSTORE_KEY);
    this.keystorePassword = context.getString(KEYSTORE_PASSWORD_KEY);

    if (enableSsl) {
        checkNotNull(keystore, KEYSTORE_KEY + " must be specified when SSL is enabled");
        checkNotNull(keystorePassword, KEYSTORE_PASSWORD_KEY + " must be specified when SSL is enabled");
    }
}
 

private synchronized RpcClient getClient(HostInfo info)
    throws FlumeException, EventDeliveryException {
  throwIfClosed();
  String name = info.getReferenceName();
  RpcClient client = clientMap.get(name);
  if (client == null) {
    client = createClient(name);
    clientMap.put(name, client);
  } else if (!client.isActive()) {
    try {
      client.close();
    } catch (Exception ex) {
      LOGGER.warn("Failed to close client for " + info, ex);
    }
    client = createClient(name);
    clientMap.put(name, client);
  }

  return client;
}
 

private void parseHostsFromString(String hosts)
        throws FlumeException {
    
    String[] hostsAndPorts = hosts.split(",");
    for (String host : hostsAndPorts) {
        String[] hostAndPort = host.split(":");
        if (hostAndPort.length < 2) {
            logger.warn("Invalid zabbix host: ", host);
            continue;
        }
        try {               
            zabbixServers.put(hostAndPort[0], Integer.parseInt(hostAndPort[1]));
        } catch (Exception e) {
            logger.warn("Invalid zabbix host: " + host, e);
            continue;
        }
    }
    if (zabbixServers.isEmpty()) {
        throw new FlumeException("No valid zabbix hosts defined!");
    }
}
 

private byte[] serialize(Object datum, Schema datumSchema) throws FlumeException {
  if (schema == null || !datumSchema.equals(schema)) {
    schema = datumSchema;
    out = new ByteArrayOutputStream();
    writer = new ReflectDatumWriter<Object>(schema);
    encoder = EncoderFactory.get().binaryEncoder(out, null);
  }
  out.reset();
  try {
    writer.write(datum, encoder);
    encoder.flush();
    return out.toByteArray();
  } catch (IOException e) {
    throw new FlumeException(e);
  }
}
 

/**
 * First connect the client, then close the client, then send a request.
 * @throws FlumeException
 * @throws EventDeliveryException
 */
@Test(expected=EventDeliveryException.class)
public void testClientClosedRequest() throws FlumeException,
    EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcTestUtils.getStockLocalClient(server.getPort());
    client.close();
    Assert.assertFalse("Client should not be active", client.isActive());
    System.out.println("Yaya! I am not active after client close!");
    client.append(EventBuilder.withBody("hello", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

@Override
public List<Row> getActions() throws FlumeException {
  List<Row> actions = new LinkedList<Row>();
  if(plCol != null){
    byte[] rowKey;
    try {
      if (keyType == KeyType.TS) {
        rowKey = SimpleRowKeyGenerator.getTimestampKey(rowPrefix);
      } else if(keyType == KeyType.RANDOM) {
        rowKey = SimpleRowKeyGenerator.getRandomKey(rowPrefix);
      } else if(keyType == KeyType.TSNANO) {
        rowKey = SimpleRowKeyGenerator.getNanoTimestampKey(rowPrefix);
      } else {
        rowKey = SimpleRowKeyGenerator.getUUIDKey(rowPrefix);
      }
      Put put = new Put(rowKey);
      put.add(cf, plCol, payload);
      actions.add(put);
    } catch (Exception e){
      throw new FlumeException("Could not get row key!", e);
    }

  }
  return actions;
}
 

/**
 * Closes underlying client.
 * If <tt>append()</tt> is called after this function is called,
 * it will throw an exception.
 * @throws FlumeException if errors occur during close
 */
@Override
public synchronized void close() throws FlumeException {
  // Any append calls after this will result in an Exception.
  if (rpcClient != null) {
    try {
      rpcClient.close();
    } catch (FlumeException ex) {
      LogLog.error("Error while trying to close RpcClient.", ex);
      if (unsafeMode) {
        return;
      }
      throw ex;
    } finally {
      rpcClient = null;
    }
  } else {
    String errorMsg = "Flume log4jappender already closed!";
    LogLog.error(errorMsg);
    if(unsafeMode) {
      return;
    }
    throw new FlumeException(errorMsg);
  }
}
 

@Test
public void testTwoParamBatchAppendOverflow() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort());
    int batchSize = client.getBatchSize();
    int moreThanBatch = batchSize + 1;
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < moreThanBatch; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

/**
 * Activate the options set using <tt>setPort()</tt>
 * and <tt>setHostname()</tt>
 *
 * @throws FlumeException if the <tt>hostname</tt> and
 *                        <tt>port</tt> combination is invalid.
 */
@Override
public void activateOptions() throws FlumeException {
  Properties props = new Properties();
  props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");
  props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",
    hostname + ":" + port);
  props.setProperty(RpcClientConfigurationConstants.CONFIG_CONNECT_TIMEOUT,
   String.valueOf(timeout));
  props.setProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT,
    String.valueOf(timeout));
  try {
    rpcClient = RpcClientFactory.getInstance(props);
    if (layout != null) {
      layout.activateOptions();
    }
  } catch (FlumeException e) {
    String errormsg = "RPC client creation failed! " +
      e.getMessage();
    LogLog.error(errormsg);
    if (unsafeMode) {
      return;
    }
    throw e;
  }
}
 

@Test(expected=FlumeException.class)
public void testCreatingLbClientSingleHost() {
  Server server1 = null;
  RpcClient c = null;
  try {
    server1 = RpcTestUtils.startServer(new OKAvroHandler());
    Properties p = new Properties();
    p.put("host1", "127.0.0.1:" + server1.getPort());
    p.put("hosts", "host1");
    p.put("client.type", "default_loadbalance");
    RpcClientFactory.getInstance(p);
  } finally {
    if (server1 != null) server1.close();
    if (c != null) c.close();
  }
}
 

/**
 * Given a list of channel names as space delimited string,
 * returns list of channels.
 * @return List of {@linkplain Channel}s represented by the names.
 */
protected List<Channel> getChannelListFromNames(String channels,
        Map<String, Channel> channelNameMap) {
  List<Channel> configuredChannels = new ArrayList<Channel>();
  if(channels == null || channels.isEmpty()) {
    return configuredChannels;
  }
  String[] chNames = channels.split(" ");
  for (String name : chNames) {
    Channel ch = channelNameMap.get(name);
    if (ch != null) {
      configuredChannels.add(ch);
    } else {
      throw new FlumeException("Selector channel not found: "
              + name);
    }
  }
  return configuredChannels;
}
 

@Test (expected = EventDeliveryException.class)
public void testTimeout() throws Throwable {
  File TESTFILE = new File(TestLoadBalancingLog4jAppender.class
    .getClassLoader()
    .getResource("flume-loadbalancinglog4jtest.properties")
    .getFile());

  ch = new TestLog4jAppender.SlowMemoryChannel(2000);
  configureChannel();
  slowDown = true;
  startSources(TESTFILE, false, new int[]{25430, 25431, 25432});
  int level = 20000;
  String msg = "This is log message number" + String.valueOf(level);
  try {
    fixture.log(Level.toLevel(level), msg);
  } catch (FlumeException ex) {
    throw ex.getCause();
  }

}
 

/**
 * Try writing to some servers and then kill them all.
 *
 * @throws FlumeException
 * @throws EventDeliveryException
 */
@Test(
    expected = EventDeliveryException.class)
public void testFailedServers() throws FlumeException, EventDeliveryException {
  FailoverRpcClient client = null;
  Server server1 = RpcTestUtils.startServer(new OKAvroHandler());
  Server server2 = RpcTestUtils.startServer(new OKAvroHandler());
  Server server3 = RpcTestUtils.startServer(new OKAvroHandler());
  Properties props = new Properties();
  props.put("client.type", "default_failover");

  props.put("hosts", "host1 host2 host3");
  props.put("hosts.host1", "localhost:" + String.valueOf(server1.getPort()));
  props.put("hosts.host2", "localhost:" + String.valueOf(server2.getPort()));
  props.put("hosts.host3", " localhost:" + String.valueOf(server3.getPort()));
  client = (FailoverRpcClient) RpcClientFactory.getInstance(props);
  List<Event> events = new ArrayList<Event>();
  for (int i = 0; i < 50; i++) {
    events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
  }
  client.appendBatch(events);
  server1.close();
  server2.close();
  server3.close();
  events = new ArrayList<Event>();
  for (int i = 0; i < 50; i++) {
    events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
  }
  client.appendBatch(events);
}
 

@Override
public Status appendBatch(List<ThriftFlumeEvent> events) throws
  TException {
  try {
    if (delay != null) {
      TimeUnit.MILLISECONDS.sleep(delay.get());
    }
  } catch (InterruptedException e) {
    throw new FlumeException("Error", e);
  }
  return super.appendBatch(events);
}
 

@Test(expected = FlumeException.class)
public void testMissingTable() throws Exception {
  deleteTable = false;
  ctx.put("batchSize", "2");
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  Assert.assertFalse(sink.isConfNull());
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 2);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 2; i++){
    for(int j = 0; j < 2; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(2, found);
  out = results[2];
  Assert.assertArrayEquals(Longs.toByteArray(2), out);
  sink.process();
  sink.stop();
}
 

private Properties getProperties(String hosts, String selector,
    String maxBackoff) throws FlumeException {

  if (StringUtils.isEmpty(hosts)) {
    throw new IllegalArgumentException("hosts must not be null");
  }

  Properties props = new Properties();
  String[] hostsAndPorts = hosts.split("\\s+");
  StringBuilder names = new StringBuilder();
  for (int i = 0; i < hostsAndPorts.length; i++) {
    String hostAndPort = hostsAndPorts[i];
    String name = "h" + i;
    props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + name,
        hostAndPort);
    names.append(name).append(" ");
  }
  props.put(RpcClientConfigurationConstants.CONFIG_HOSTS, names.toString());
  props.put(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
      ClientType.DEFAULT_LOADBALANCE.toString());
  if (!StringUtils.isEmpty(selector)) {
    props.put(RpcClientConfigurationConstants.CONFIG_HOST_SELECTOR, selector);
  }

  if (!StringUtils.isEmpty(maxBackoff)) {
    long millis = Long.parseLong(maxBackoff.trim());
    if (millis <= 0) {
      throw new IllegalArgumentException(
          "Misconfigured max backoff, value must be greater than 0");
    }
    props.put(RpcClientConfigurationConstants.CONFIG_BACKOFF,
        String.valueOf(true));
    props.put(RpcClientConfigurationConstants.CONFIG_MAX_BACKOFF, maxBackoff);
  }
  return props;
}
 

@Override
public Status append(ThriftFlumeEvent event) throws TException {
  try {
    TimeUnit.MILLISECONDS.sleep(1550);
  } catch (InterruptedException e) {
    throw new FlumeException("Error", e);
  }
  return super.append(event);
}
 

@Override
public synchronized void append(LoggingEvent event) {
  if(!configured) {
    String errorMsg = "Flume Log4jAppender not configured correctly! Cannot" +
      " send events to Flume.";
    LogLog.error(errorMsg);
    if(getUnsafeMode()) {
      return;
    }
    throw new FlumeException(errorMsg);
  }
  super.append(event);
}
 

@Test
public void testRandomBackoff() throws Exception {
  File TESTFILE = new File(TestLoadBalancingLog4jAppender.class
      .getClassLoader()
      .getResource("flume-loadbalancing-backoff-log4jtest.properties")
      .getFile());
  startSources(TESTFILE, new int[] { 25430, 25431, 25432 });

  sources.get(0).setFail();
  sources.get(2).setFail();

  sendAndAssertMessages(50);

  Assert.assertEquals(50, sources.get(1).appendCount.intValue());
  Assert.assertEquals(0, sources.get(0).appendCount.intValue());
  Assert.assertEquals(0, sources.get(2).appendCount.intValue());
  sources.get(0).setOk();
  sources.get(1).setFail(); // s0 should still be backed off
  try {
    send(1);
    // nothing should be able to process right now
    Assert.fail("Expected EventDeliveryException");
  } catch (FlumeException e) {
    Assert.assertTrue(e.getCause() instanceof EventDeliveryException);
  }
  Thread.sleep(2500); // wait for s0 to no longer be backed off

  sendAndAssertMessages(50);

  Assert.assertEquals(50, sources.get(0).appendCount.intValue());
  Assert.assertEquals(50, sources.get(1).appendCount.intValue());
  Assert.assertEquals(0, sources.get(2).appendCount.intValue());
}
 

public synchronized void supervise(LifecycleAware lifecycleAware,
    SupervisorPolicy policy, LifecycleState desiredState) {
  if(this.monitorService.isShutdown()
      || this.monitorService.isTerminated()
      || this.monitorService.isTerminating()){
    throw new FlumeException("Supervise called on " + lifecycleAware + " " +
        "after shutdown has been initiated. " + lifecycleAware + " will not" +
        " be started");
  }

  Preconditions.checkState(!supervisedProcesses.containsKey(lifecycleAware),
      "Refusing to supervise " + lifecycleAware + " more than once");

  if (logger.isDebugEnabled()) {
    logger.debug("Supervising service:{} policy:{} desiredState:{}",
        new Object[] { lifecycleAware, policy, desiredState });
  }

  Supervisoree process = new Supervisoree();
  process.status = new Status();

  process.policy = policy;
  process.status.desiredState = desiredState;
  process.status.error = false;

  MonitorRunnable monitorRunnable = new MonitorRunnable();
  monitorRunnable.lifecycleAware = lifecycleAware;
  monitorRunnable.supervisoree = process;
  monitorRunnable.monitorService = monitorService;

  supervisedProcesses.put(lifecycleAware, process);

  ScheduledFuture<?> future = monitorService.scheduleWithFixedDelay(
      monitorRunnable, 0, 3, TimeUnit.SECONDS);
  monitorFutures.put(lifecycleAware, future);
}
 

private TailFile openFile(File file, Map<String, String> headers, long inode, long pos, long lineNum) {
    try {
        logger.info("Opening file: " + file + ", inode: " + inode + ", pos: " + pos + ", lineNum：" + lineNum);
        return new TailFile(file, headers, inode, pos, lineNum);
    } catch (IOException e) {
        throw new FlumeException("Failed opening file: " + file, e);
    }
}
 

@Test
public void testTwoParamDeprecatedAppend() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getInstance(localhost, server.getPort());
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 

/**
 * Closes underlying client.
 * If <tt>append()</tt> is called after this function is called,
 * it will throw an exception.
 * @throws FlumeException if errors occur during close
 */
@Override
public synchronized void close() throws FlumeException{
  //Any append calls after this will result in an Exception.
  if (rpcClient != null) {
    rpcClient.close();
    rpcClient = null;
  }
}
 

@Override
public synchronized void start(){
	logger.info("SpoolDirectoryTailFileSource source starting with directory: {}, targetFilename: {}", spoolDirectory, targetFilename);
	
	executor = Executors.newSingleThreadScheduledExecutor();
	File directory = new File(spoolDirectory);
	
	try {
		reader = (new ReliableSpoolDirectoryTailFileEventReader.Builder())
						.spoolDirectory(directory)
						.completedSuffix(completedSuffix)
						.ignorePattern(ignorePattern)
						.targetPattern(targetPattern)
						.targetFilename(targetFilename)
						.trackerDirPath(trackerDirPath)
						.annotateFileName(fileHeader)
						.fileNameHeader(fileHeaderKey)
						.annotateBaseName(basenameHeader)
						.baseNameHeader(basenameHeaderKey)
						.deserializerType(deserializerType)
						.deserializerContext(deserializerContext)
						.deletePolicy(deletePolicy)
						.inputCharset(inputCharset)
						.decodeErrorPolicy(decodeErrorPolicy)
						.consumeOrder(consumeOrder)
						.build();
	} catch (IOException e) {
		throw new FlumeException("Error instantiating spooling and tail event parser", e);
	}
	
	Runnable runner = new SpoolDirectoryTailFileRunnable(reader, sourceCounter);
	executor.scheduleWithFixedDelay(runner, 0, POLL_DELAY_MS, TimeUnit.MILLISECONDS);
	
	super.start();
	logger.debug("SpoolDirectoryTailFileSource source started");
	sourceCounter.start();
	
}
 

@Test(expected = FlumeException.class)
public void testMissingTable() throws Exception {
  ctx.put("batchSize", "2");
  HBaseSink sink = new HBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 2);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 2; i++){
    for(int j = 0; j < 2; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(2, found);
  out = results[2];
  Assert.assertArrayEquals(Longs.toByteArray(2), out);
  sink.process();
  sink.stop();
}