org.apache.flume.source.avro.AvroFlumeEvent Java Examples

private byte[] serializeEvent(Event event, boolean useAvroEventFormat) throws IOException {
    byte[] bytes;
    if (useAvroEventFormat) {
        if (!tempOutStream.isPresent()) {
            tempOutStream = Optional.of(new ByteArrayOutputStream());
        }
        if (!writer.isPresent()) {
            writer = Optional.of(new SpecificDatumWriter<AvroFlumeEvent>(AvroFlumeEvent.class));
        }
        tempOutStream.get().reset();
        AvroFlumeEvent e = new AvroFlumeEvent(toCharSeqMap(event.getHeaders()),
                ByteBuffer.wrap(event.getBody()));
        encoder = EncoderFactory.get().directBinaryEncoder(tempOutStream.get(), encoder);
        writer.get().write(e, encoder);
        encoder.flush();
        bytes = tempOutStream.get().toByteArray();
    } else {
        bytes = event.getBody();
    }
    return bytes;
}
 

@Override
public Status append(AvroFlumeEvent avroEvent) {
  logger.debug("Avro source {}: Received avro event: {}", getName(),
      avroEvent);
  sourceCounter.incrementAppendReceivedCount();
  sourceCounter.incrementEventReceivedCount();

  Event event = EventBuilder.withBody(avroEvent.getBody().array(),
      toStringMap(avroEvent.getHeaders()));

  try {
    getChannelProcessor().processEvent(event);
  } catch (ChannelException ex) {
    logger.warn("Avro source " + getName() + ": Unable to process event. " +
        "Exception follows.", ex);
    return Status.FAILED;
  }

  sourceCounter.incrementAppendAcceptedCount();
  sourceCounter.incrementEventAcceptedCount();

  return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) {
  if (isFail) {
    return Status.FAILED;
  }
  appendCount.addAndGet(events.size());
  return super.appendBatch(events);
}
 

public Event poll() {
  AvroFlumeEvent avroEvent = eventQueue.poll();
  if(avroEvent != null) {
    return EventBuilder.withBody(avroEvent.getBody().array(),
        toStringMap(avroEvent.getHeaders()));
  }
  return null;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) {
  if (isFail) {
    return Status.FAILED;
  }
  appendCount.addAndGet(events.size());
  return super.appendBatch(events);
}
 

@Override
public Status append(AvroFlumeEvent avroEvent) {
  if (isFail) {
    return Status.FAILED;
  }
  appendCount.incrementAndGet();
  return super.append(avroEvent);
}
 

public Event poll() {

            AvroFlumeEvent avroEvent = null;
            try {
                avroEvent = eventQueue.poll(30000, TimeUnit.MILLISECONDS);
            } catch (final InterruptedException ie) {
                // Ignore the exception.
            }
            if (avroEvent != null) {
                return EventBuilder.withBody(avroEvent.getBody().array(), toStringMap(avroEvent.getHeaders()));
            }
            System.out.println("No Event returned");
            return null;
        }
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events)
    throws AvroRemoteException {
  logger.debug("Received event batch:{}; delaying for {}ms", events, delay);
  sleep();
  return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) {
  logger.debug("Avro source {}: Received avro event batch of {} events.",
      getName(), events.size());
  sourceCounter.incrementAppendBatchReceivedCount();
  sourceCounter.addToEventReceivedCount(events.size());

  List<Event> batch = new ArrayList<Event>();

  for (AvroFlumeEvent avroEvent : events) {
    Event event = EventBuilder.withBody(avroEvent.getBody().array(),
        toStringMap(avroEvent.getHeaders()));

    batch.add(event);
  }

  try {
    getChannelProcessor().processEventBatch(batch);
  } catch (Throwable t) {
    logger.error("Avro source " + getName() + ": Unable to process event " +
        "batch. Exception follows.", t);
    if (t instanceof Error) {
      throw (Error) t;
    }
    return Status.FAILED;
  }

  sourceCounter.incrementAppendBatchAcceptedCount();
  sourceCounter.addToEventAcceptedCount(events.size());

  return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  logger.info("Throwing: Received {} events from appendBatch()",
      events.size());
  throw new AvroRemoteException("Handler smash!");
}
 

public Event poll() {

            AvroFlumeEvent avroEvent = null;
            try {
                avroEvent = eventQueue.poll(30000, TimeUnit.MILLISECONDS);
            } catch (final InterruptedException ie) {
                // Ignore the exception.
            }
            if (avroEvent != null) {
                return EventBuilder.withBody(avroEvent.getBody().array(), toStringMap(avroEvent.getHeaders()));
            }
            System.out.println("No Event returned");
            return null;
        }
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  logger.info("Unknown: Received {} events from appendBatch()",
      events.size());
  return Status.UNKNOWN;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  logger.info("Failed: Received {} events from appendBatch()",
      events.size());
  return Status.FAILED;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  logger.info("OK: Received {} events from appendBatch()",
      events.size());
  return Status.OK;
}
 

public Event poll() {

            AvroFlumeEvent avroEvent = null;
            try {
                avroEvent = eventQueue.poll(30000, TimeUnit.MILLISECONDS);
            } catch (final InterruptedException ie) {
                // Ignore the exception.
            }
            if (avroEvent != null) {
                return EventBuilder.withBody(avroEvent.getBody().array(), toStringMap(avroEvent.getHeaders()));
            }
            System.out.println("No Event returned");
            return null;
        }
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events) throws
    AvroRemoteException {
  if (failed) {
    logger.debug("Event batch rejected");
    return Status.FAILED;
  }
  logger.debug("LB: Received {} events from appendBatch()",
      events.size());

  appendBatchCount++;
  return Status.OK;
}
 

@Override
public Status append(AvroFlumeEvent event) throws AvroRemoteException {
  if (failed) {
    logger.debug("Event rejected");
    return Status.FAILED;
  }
  logger.debug("LB: Received event from append(): {}",
      new String(event.getBody().array(), Charset.forName("UTF8")));
  appendCount++;
  return Status.OK;
}
 

@Override
public Status append(AvroFlumeEvent avroEvent) {
  if (isFail) {
    return Status.FAILED;
  }
  appendCount.incrementAndGet();
  return super.append(avroEvent);
}
 

private void start() {
  flumeServer = RpcTestUtils.startServer(new AvroSourceProtocol(){

    @Override
    public Status append(AvroFlumeEvent event) throws AvroRemoteException {
      return protocol.append(event);
    }

    @Override
    public Status appendBatch(List<AvroFlumeEvent> events) throws AvroRemoteException {
      return protocol.appendBatch(events);
    }
  });
}
 

@Override
public Status appendBatch(final List<AvroFlumeEvent> events) throws AvroRemoteException {
    Preconditions.checkState(eventQueue.addAll(events));
    for (final AvroFlumeEvent event : events) {
        // System.out.println("Received event " + event.getHeaders().get(new org.apache.avro.util.Utf8(FlumeEvent.GUID)));
    }
    return Status.OK;
}
 

public Event poll() {

            AvroFlumeEvent avroEvent = null;
            try {
                avroEvent = eventQueue.poll(30000, TimeUnit.MILLISECONDS);
            } catch (final InterruptedException ie) {
                // Ignore the exception.
            }
            if (avroEvent != null) {
                return EventBuilder.withBody(avroEvent.getBody().array(), toStringMap(avroEvent.getHeaders()));
            }
            System.out.println("No Event returned");
            return null;
        }
 

@Override
public Status append(AvroFlumeEvent event) throws AvroRemoteException {
  logger.debug("Received event:{}; delaying for {}ms", event, delay);
  sleep();
  return Status.OK;
}
 

@Override
public Status appendBatch(final List<AvroFlumeEvent> events)
    throws AvroRemoteException {
    Preconditions.checkState(eventQueue.addAll(events));
    return Status.OK;
}
 

@Override
public Status appendBatch(List<AvroFlumeEvent> events)
    throws AvroRemoteException {
  Preconditions.checkState(eventQueue.addAll(events));
  return Status.OK;
}
 

@Override
public Status append(final AvroFlumeEvent event) throws AvroRemoteException {
    eventQueue.add(event);
    //System.out.println("Received event " + event.getHeaders().get(new org.apache.avro.util.Utf8(FlumeEvent.GUID)));
    return Status.OK;
}
 

@Override
public Status append(final AvroFlumeEvent event) throws AvroRemoteException {
    eventQueue.add(event);
    return Status.OK;
}
 

@Override
public Status appendBatch(final List<AvroFlumeEvent> events)
    throws AvroRemoteException {
    Preconditions.checkState(eventQueue.addAll(events));
    return Status.OK;
}
 

@Override
public Status appendBatch(final List<AvroFlumeEvent> events)
    throws AvroRemoteException {
    Preconditions.checkState(eventQueue.addAll(events));
    return Status.OK;
}
 

@Override
public Status append(final AvroFlumeEvent event) throws AvroRemoteException {
    eventQueue.add(event);
    return Status.OK;
}
 

@Override
public Status append(final AvroFlumeEvent event) throws AvroRemoteException {
    eventQueue.add(event);
    return Status.OK;
}