org.apache.flume.ChannelException Java Examples

@Test
public void testReconfigure() throws Exception {
  channel.start();
  Assert.assertTrue(channel.isOpen());
  Set<String> in = Sets.newHashSet();
  try {
    while(true) {
      in.addAll(putEvents(channel, "reconfig", 1, 1));
    }
  } catch (ChannelException e) {
    Assert.assertEquals("The channel has reached it's capacity. "
        + "This might be the result of a sink on the channel having too "
        + "low of batch size, a downstream system running slower than "
        + "normal, or that the channel capacity is just too low. [channel="
        + channel.getName()+"]", e.getMessage());
  }
  Configurables.configure(channel, createContext());
  Set<String> out = takeEvents(channel, 1, Integer.MAX_VALUE);
  compareInputAndOut(in, out);
}
 

@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent mEvent) {
  ChannelBuffer buff = (ChannelBuffer) mEvent.getMessage();
  while (buff.readable()) {
    Event e = syslogUtils.extractEvent(buff);
    if (e == null) {
      logger.debug("Parsed partial event, event will be generated when " +
          "rest of the event is received.");
      continue;
    }
    try {
      getChannelProcessor().processEvent(e);
      counterGroup.incrementAndGet("events.success");
    } catch (ChannelException ex) {
      counterGroup.incrementAndGet("events.dropped");
      logger.error("Error writting to channel, event dropped", ex);
    }
  }

}
 

@Override
public Status append(ThriftFlumeEvent event) throws TException {
  Event flumeEvent = EventBuilder.withBody(event.getBody(),
    event.getHeaders());

  sourceCounter.incrementAppendReceivedCount();
  sourceCounter.incrementEventReceivedCount();

  try {
    getChannelProcessor().processEvent(flumeEvent);
  } catch (ChannelException ex) {
    logger.warn("Thrift source " + getName() + " could not append events " +
      "to the channel.", ex);
    return Status.FAILED;
  }
  sourceCounter.incrementAppendAcceptedCount();
  sourceCounter.incrementEventAcceptedCount();
  return Status.OK;
}
 

@Override
public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
  sourceCounter.incrementAppendBatchReceivedCount();
  sourceCounter.addToEventReceivedCount(events.size());

  List<Event> flumeEvents = Lists.newArrayList();
  for(ThriftFlumeEvent event : events) {
    flumeEvents.add(EventBuilder.withBody(event.getBody(),
      event.getHeaders()));
  }

  try {
    getChannelProcessor().processEventBatch(flumeEvents);
  } catch (ChannelException ex) {
    logger.warn("Thrift source %s could not append events to the " +
      "channel.", getName());
    return Status.FAILED;
  }

  sourceCounter.incrementAppendBatchAcceptedCount();
  sourceCounter.addToEventAcceptedCount(events.size());
  return Status.OK;
}
 

@Override
public Status append(AvroFlumeEvent avroEvent) {
  logger.debug("Avro source {}: Received avro event: {}", getName(),
      avroEvent);
  sourceCounter.incrementAppendReceivedCount();
  sourceCounter.incrementEventReceivedCount();

  Event event = EventBuilder.withBody(avroEvent.getBody().array(),
      toStringMap(avroEvent.getHeaders()));

  try {
    getChannelProcessor().processEvent(event);
  } catch (ChannelException ex) {
    logger.warn("Avro source " + getName() + ": Unable to process event. " +
        "Exception follows.", ex);
    return Status.FAILED;
  }

  sourceCounter.incrementAppendAcceptedCount();
  sourceCounter.incrementEventAcceptedCount();

  return Status.OK;
}
 

/**
 * <p>
 * The method to which {@link BasicChannelSemantics} delegates calls
 * to <code>put</code>.
 * </p>
 */
protected void put(Event event) {
  Preconditions.checkState(Thread.currentThread().getId() == initialThreadId,
      "put() called from different thread than getTransaction()!");
  Preconditions.checkState(state.equals(State.OPEN),
      "put() called when transaction is %s!", state);
  Preconditions.checkArgument(event != null,
      "put() called with null event!");

  try {
    doPut(event);
  } catch (InterruptedException e) {
    Thread.currentThread().interrupt();
    throw new ChannelException(e.toString(), e);
  }
}
 

/**
 * <p>
 * A convenience method for multiple-event <code>take</code> transactions.
 * </p>
 * @return a list of at most <code>max</code> events
 * @see #transact(Channel,Callable)
 */
public static List<Event> take(final Channel channel, final int max)
    throws ChannelException {
  return transact(channel, new Callable<List<Event>>() {
      @Override
      public List<Event> call() {
        List<Event> events = new ArrayList<Event>(max);
        while (events.size() < max) {
          Event event = channel.take();
          if (event == null) {
            break;
          }
          events.add(event);
        }
        return events;
      }
    });
}
 

@Override
protected void doPut(Event event) throws InterruptedException {
  channelCounter.incrementEventPutAttemptCount();
  int eventByteSize = (int)Math.ceil(estimateEventSize(event)/byteCapacitySlotSize);

  if (bytesRemaining.tryAcquire(eventByteSize, keepAlive, TimeUnit.SECONDS)) {
    if(!putList.offer(event)) {
      throw new ChannelException("Put queue for MemoryTransaction of capacity " +
          putList.size() + " full, consider committing more frequently, " +
          "increasing capacity or increasing thread count");
    }
  } else {
    throw new ChannelException("Put queue for MemoryTransaction of byteCapacity " +
        (lastByteCapacity * (int)byteCapacitySlotSize) + " bytes cannot add an " +
        " event of size " + estimateEventSize(event) + " bytes because " +
         (bytesRemaining.availablePermits() * (int)byteCapacitySlotSize) + " bytes are already used." +
        " Try consider comitting more frequently, increasing byteCapacity or increasing thread count");
  }
  putByteCounter += eventByteSize;
}
 

@Test
public void testEncryptedChannelWithoutEncryptionConfigFails() throws Exception {
  Map<String, String> overrides = getOverridesForEncryption();
  channel = createFileChannel(overrides);
  channel.start();
  Assert.assertTrue(channel.isOpen());
  fillChannel(channel, "will-not-restart");
  channel.stop();
  Map<String, String> noEncryptionOverrides = getOverrides();
  channel = createFileChannel(noEncryptionOverrides);
  channel.start();

  if(channel.isOpen()) {
    try {
      takeEvents(channel, 1, 1);
      Assert.fail("Channel was opened and take did not throw exception");
    } catch(ChannelException ex) {
      // expected
    }
  }
}
 

/**
 * Ensure that we bubble up any specific exception thrown from getTransaction
 * instead of another exception masking it such as an NPE
 */
@Test(expected = ChannelException.class)
public void testExceptionFromGetTransaction() {
  // create a channel which unexpectedly throws a ChEx on getTransaction()
  Channel ch = mock(Channel.class);
  when(ch.getTransaction()).thenThrow(new ChannelException("doh!"));

  ChannelSelector sel = new ReplicatingChannelSelector();
  sel.setChannels(Lists.newArrayList(ch));
  ChannelProcessor proc = new ChannelProcessor(sel);

  List<Event> events = Lists.newArrayList();
  events.add(EventBuilder.withBody("event 1", Charsets.UTF_8));

  proc.processEventBatch(events);
}
 

@Test(expected=ChannelException.class)
public void testTransactionPutCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "2");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  // shouldn't be able to fit a third in the buffer
  channel.put(EventBuilder.withBody("test".getBytes()));
  Assert.fail();
}
 

@Test(expected=ChannelException.class)
public void testCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "3");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  // this should kill  it
  transaction.commit();
  Assert.fail();
}
 

@Override
public Void append( AvroFlumeOGEvent evt ) throws AvroRemoteException {
  counterGroup.incrementAndGet("rpc.received");
  Map<String, String> headers = new HashMap<String, String>();

  // extract Flume OG event headers
  headers.put(HOST, evt.getHost().toString());
  headers.put(TIMESTAMP, evt.getTimestamp().toString());
  headers.put(PRIORITY, evt.getPriority().toString());
  headers.put(NANOS, evt.getNanos().toString());
  for (Entry<CharSequence, ByteBuffer> entry : evt.getFields().entrySet()) {
    headers.put(entry.getKey().toString(), entry.getValue().toString());
  }
  headers.put(OG_EVENT, "yes");

  Event event = EventBuilder.withBody(evt.getBody().array(), headers);
  try {
    getChannelProcessor().processEvent(event);
    counterGroup.incrementAndGet("rpc.events");
  } catch (ChannelException ex) {
    return null;
  }

  counterGroup.incrementAndGet("rpc.successful");
  return null;
}
 

@Override
protected Event doTake() throws InterruptedException {
  channelCounter.incrementEventTakeAttemptCount();
  if(takeList.remainingCapacity() == 0) {
    throw new ChannelException("Take list for MemoryTransaction, capacity " +
        takeList.size() + " full, consider committing more frequently, " +
        "increasing capacity, or increasing thread count");
  }
  if(!queueStored.tryAcquire(keepAlive, TimeUnit.SECONDS)) {
    return null;
  }
  Event event;
  synchronized(queueLock) {
    event = queue.poll();
  }
  Preconditions.checkNotNull(event, "Queue.poll returned NULL despite semaphore " +
      "signalling existence of entry");
  takeList.put(event);

  int eventByteSize = (int)Math.ceil(estimateEventSize(event)/byteCapacitySlotSize);
  takeByteCounter += eventByteSize;

  return event;
}
 

@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent mEvent) {
  try {
    syslogUtils.setEventSize(maxsize);
    Event e = syslogUtils.extractEvent((ChannelBuffer)mEvent.getMessage());
    if (e == null) {
      return;
    }
    getChannelProcessor().processEvent(e);
    counterGroup.incrementAndGet("events.success");
  } catch (ChannelException ex) {
    counterGroup.incrementAndGet("events.dropped");
    logger.error("Error writting to channel", ex);
    return;
  }
}
 

public static Set<String> putEvents(Channel channel, String prefix,
        int batchSize, int numEvents, boolean untilCapacityIsReached)
            throws Exception {
  Set<String> result = Sets.newHashSet();
  for (int i = 0; i < numEvents; i += batchSize) {
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    try {
      Set<String> batch = Sets.newHashSet();
      for (int j = 0; j < batchSize; j++) {
        String s = prefix + "-" + i + "-" + j + "-" + UUID.randomUUID();
        Event event = EventBuilder.withBody(s.getBytes(Charsets.UTF_8));
        channel.put(event);
        batch.add(s);
      }
      transaction.commit();
      result.addAll(batch);
    } catch (Exception ex) {
      transaction.rollback();
      if(untilCapacityIsReached && ex instanceof ChannelException &&
          ("The channel has reached it's capacity. "
              + "This might be the result of a sink on the channel having too "
              + "low of batch size, a downstream system running slower than "
              + "normal, or that the channel capacity is just too low. "
              + "[channel=" +channel.getName() + "]").
            equals(ex.getMessage())) {
        break;
      }
      throw ex;
    } finally {
      transaction.close();
    }
  }
  return result;
}
 

@Override
public Status process() throws EventDeliveryException {
  long totalEventSent = counterGroup.addAndGet("events.total", lastSent);

  if ((maxTotalEvents >= 0 &&
      totalEventSent >= maxTotalEvents) ||
      (maxSuccessfulEvents >= 0 &&
      counterGroup.get("events.successful") >= maxSuccessfulEvents)) {
    return Status.BACKOFF;
  }
  try {
    lastSent = batchSize;

    if (batchSize == 1) {
      getChannelProcessor().processEvent(event);
    } else {
      long eventsLeft = maxTotalEvents - totalEventSent;

      if (maxTotalEvents >= 0 && eventsLeft < batchSize) {
        eventBatchListToProcess = eventBatchList.subList(0, (int)eventsLeft);
      } else {
        eventBatchListToProcess = eventBatchList;
      }
      lastSent = eventBatchListToProcess.size();
      getChannelProcessor().processEventBatch(eventBatchListToProcess);
    }

    counterGroup.addAndGet("events.successful", lastSent);
  } catch (ChannelException ex) {
    counterGroup.addAndGet("events.failed", lastSent);
    return Status.BACKOFF;
  }
  return Status.READY;
}
 

private List<Event> takeEventsFromChannel(Channel channel, long eventsToTake) throws ChannelException {
    List<Event> events = new ArrayList<Event>();
    Event event;
    for (int i = 0; i < eventsToTake; i++) {
        event = buildEvent(channel);
        events.add(event);
        if (event != null) {
            sinkCounter.incrementEventDrainAttemptCount();
        }
    }
    events.removeAll(Collections.singleton(null));
    return events;
}
 

@Override
public Event take() {
  Preconditions.checkState(queue != null,
      "No queue defined (Did you forget to configure me?");
  channelCounter.incrementEventTakeAttemptCount();
  try {
    Event e = queue.poll(keepAlive, TimeUnit.SECONDS);
    channelCounter.addToEventTakeSuccessCount(1);
    channelCounter.setChannelSize(queue.size());
    return e;
  } catch (InterruptedException ex) {
    throw new ChannelException("Failed to take()", ex);
  }
}
 

@Override
protected void doPut(Event event) throws InterruptedException {
    type = TransactionType.PUT;
    if (!serializedEvents.isPresent()) {
        serializedEvents = Optional.of(new LinkedList<byte[]>());
    }

    try {
        serializedEvents.get().add(event.getBody());
    } catch (Exception e) {
        throw new ChannelException("Error while serializing event", e);
    }
}
 

@Override
public Status process() throws EventDeliveryException {

  Status status = Status.READY;
  int i = 0;
  try {
    if (batchSize <= 1) {
      if(eventsSent < totalEvents) {
        getChannelProcessor().processEvent(
          EventBuilder.withBody(String.valueOf(sequence++).getBytes()));
        sourceCounter.incrementEventAcceptedCount();
        eventsSent++;
      } else {
        status = Status.BACKOFF;
      }
    } else {
      batchArrayList.clear();
      for (i = 0; i < batchSize; i++) {
        if(eventsSent < totalEvents){
          batchArrayList.add(i, EventBuilder.withBody(String
            .valueOf(sequence++).getBytes()));
          eventsSent++;
        } else {
          status = Status.BACKOFF;
        }
      }
      if(!batchArrayList.isEmpty()) {
        getChannelProcessor().processEventBatch(batchArrayList);
        sourceCounter.incrementAppendBatchAcceptedCount();
        sourceCounter.addToEventAcceptedCount(batchArrayList.size());
      }
    }

  } catch (ChannelException ex) {
    eventsSent -= i;
    logger.error( getName() + " source could not write to channel.", ex);
  }

  return status;
}
 

/**
 * Roll a log if needed. Roll always occurs if the log at the index
 * does not exist (typically on startup), or buffer is null. Otherwise
 * LogFile.Writer.isRollRequired is checked again to ensure we don't
 * have threads pile up on this log resulting in multiple successive
 * rolls
 *
 * Synchronization required since both synchronized and unsynchronized
 * methods call this method, and this method acquires only a
 * read lock. The synchronization guarantees that multiple threads don't
 * roll at the same time.
 *
 * @param index
 * @throws IOException
 */
  private synchronized void roll(int index, ByteBuffer buffer)
    throws IOException {
  if (!tryLockShared()) {
    throw new ChannelException("Failed to obtain lock for writing to the "
        + "log. Try increasing the log write timeout value. " +
        channelNameDescriptor);
  }

  try {
    LogFile.Writer oldLogFile = logFiles.get(index);
    // check to make sure a roll is actually required due to
    // the possibility of multiple writes waiting on lock
    if(oldLogFile == null || buffer == null ||
        oldLogFile.isRollRequired(buffer)) {
      try {
        LOGGER.info("Roll start " + logDirs[index]);
        int fileID = nextFileID.incrementAndGet();
        File file = new File(logDirs[index], PREFIX + fileID);
        LogFile.Writer writer = LogFileFactory.getWriter(file, fileID,
            maxFileSize, encryptionKey, encryptionKeyAlias,
            encryptionCipherProvider, usableSpaceRefreshInterval);
        idLogFileMap.put(fileID, LogFileFactory.getRandomReader(file,
            encryptionKeyProvider));
        // writer from this point on will get new reference
        logFiles.set(index, writer);
        // close out old log
        if (oldLogFile != null) {
          oldLogFile.close();
        }
      } finally {
        LOGGER.info("Roll end");
      }
    }
  } finally {
    unlockShared();
  }
}
 

@Override
protected void doRollback() throws InterruptedException {
  int puts = putList.size();
  int takes = takeList.size();
  boolean lockAcquired = log.tryLockShared();
  try {
    if(!lockAcquired) {
      throw new ChannelException("Failed to obtain lock for writing to the "
          + "log. Try increasing the log write timeout value. " +
          channelNameDescriptor);
    }
    if(takes > 0) {
      Preconditions.checkState(puts == 0, "nonzero puts and takes "
          + channelNameDescriptor);
      synchronized (queue) {
        while (!takeList.isEmpty()) {
          Preconditions.checkState(queue.addHead(takeList.removeLast()),
              "Queue add failed, this shouldn't be able to happen "
                  + channelNameDescriptor);
        }
      }
    }
    putList.clear();
    takeList.clear();
    queue.completeTransaction(transactionID);
    channelCounter.setChannelSize(queue.getSize());
    log.rollback(transactionID);
  } catch (IOException e) {
    throw new ChannelException("Commit failed due to IO error "
        + channelNameDescriptor, e);
  } finally {
    if(lockAcquired) {
      log.unlockShared();
    }
    // since rollback is being called, puts will never make it on
    // to the queue and we need to be sure to release the resources
    queueRemaining.release(puts);
  }
}
 

protected void doMode() throws InterruptedException {
  switch (mode) {
    case THROW_ERROR:
      throw new TestError();
    case THROW_RUNTIME:
      throw new TestRuntimeException();
    case THROW_CHANNEL:
      throw new ChannelException("test");
    case SLEEP:
      Thread.sleep(300000);
      break;
  }
}
 

protected void testException(Class<? extends Throwable> exceptionClass,
    Runnable test) {
  try {
    test.run();
    Assert.fail();
  } catch (Throwable e) {
    if (exceptionClass == InterruptedException.class
        && e instanceof ChannelException
        && e.getCause() instanceof InterruptedException) {
      Assert.assertTrue(Thread.interrupted());
    } else if (!exceptionClass.isInstance(e)) {
      throw new AssertionError(e);
    }
  }
}
 

@Test(expected=ChannelException.class)
public void testByteCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("byteCapacity", "2000");
  parms.put("byteCapacityBufferPercentage", "20");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  byte[] eventBody = new byte[405];

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  // this should kill  it
  transaction.commit();
  Assert.fail();

}
 

@Override
public Event take() throws ChannelException {
  if (events.size() > 0) {
    return events.remove(0);
  }
  return null;
}
 

@Test
public void testMaxSuccessfulEvents() throws InterruptedException,
    EventDeliveryException {
  StressSource source = new StressSource();
  source.setChannelProcessor(mockProcessor);
  Context context = new Context();
  context.put("maxSuccessfulEvents", "35");
  source.configure(context);

  for (int i = 0; i < 10; i++) {
    source.process();
  }

  // 1 failed call, 10 successful
  doThrow(new ChannelException("stub")).when(
      mockProcessor).processEvent(getEvent(source));
  source.process();
  doNothing().when(mockProcessor).processEvent(getEvent(source));
  for (int i = 0; i < 10; i++) {
    source.process();
  }

  // 1 failed call, 50 succesful
  doThrow(new ChannelException("stub")).when(
      mockProcessor).processEvent(getEvent(source));
  source.process();
  doNothing().when(mockProcessor).processEvent(getEvent(source));
  for (int i = 0; i < 50; i++) {
    source.process();
  }

  // We should have called processEvent(evt) 37 times, twice for failures
  // and twice for successful events.
  verify(mockProcessor, times(37)).processEvent(getEvent(source));
}
 

@SuppressWarnings("unchecked")
@Test
public void testProcessChannelProcessorThrowsChannelException() throws Exception {
  doThrow(new ChannelException("dummy"))
    .when(channelProcessor).processEventBatch(any(List.class));
  source.configure(context);
  source.start();
  Assert.assertEquals(Status.BACKOFF, source.process());
  verify(consumer).rollback();
}
 

private void checkIfChannelExceptionAndThrow(Throwable e)
        throws EventDeliveryException {
  if (e instanceof ChannelException) {
    throw new EventDeliveryException("Error in processing transaction.", e);
  } else if (e instanceof Error || e instanceof RuntimeException) {
    Throwables.propagate(e);
  }
  throw new EventDeliveryException("Error in processing transaction.", e);
}